From 90ae100cf482529828febed591172433309bc12e Mon Sep 17 00:00:00 2001 From: Bjorn Lu Date: Thu, 24 Oct 2024 17:13:22 +0800 Subject: [PATCH] Unset charset=utf-8 content-type for md/mdx pages (#12231) Co-authored-by: Sarah Rainsberger --- .changeset/dirty-cooks-explode.md | 9 +++ .changeset/strong-months-grab.md | 11 +++ .../core/preview/vite-plugin-astro-preview.ts | 2 +- .../routing/astro-designed-error-pages.ts | 2 +- .../astro/src/runtime/server/render/page.ts | 7 +- .../src/vite-plugin-astro-server/response.ts | 2 +- .../astro/src/vite-plugin-markdown/index.ts | 8 +- packages/astro/test/astro-basic.test.js | 25 +----- packages/integrations/mdx/src/index.ts | 13 ++-- .../src/rehype-apply-frontmatter-export.ts | 76 ++++++++++++++++++- .../integrations/mdx/src/vite-plugin-mdx.ts | 13 +++- .../mdx-page/src/layouts/EncodingLayout.astro | 1 + .../chinese-encoding-layout-frontmatter.mdx | 7 ++ .../pages/chinese-encoding-layout-manual.mdx | 12 +++ .../mdx-page/src/pages/chinese-encoding.mdx} | 0 .../integrations/mdx/test/mdx-page.test.js | 44 +++++++++++ 16 files changed, 188 insertions(+), 44 deletions(-) create mode 100644 .changeset/dirty-cooks-explode.md create mode 100644 .changeset/strong-months-grab.md create mode 100644 packages/integrations/mdx/test/fixtures/mdx-page/src/layouts/EncodingLayout.astro create mode 100644 packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-frontmatter.mdx create mode 100644 packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-manual.mdx rename packages/{astro/test/fixtures/astro-basic/src/pages/chinese-encoding-mdx.mdx => integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding.mdx} (100%) diff --git a/.changeset/dirty-cooks-explode.md b/.changeset/dirty-cooks-explode.md new file mode 100644 index 000000000000..770ef17999d9 --- /dev/null +++ b/.changeset/dirty-cooks-explode.md @@ -0,0 +1,9 @@ +--- +'@astrojs/mdx': major +--- + +Handles the breaking change in Astro where content pages (including `.mdx` pages located within `src/pages/`) no longer respond with `charset=utf-8` in the `Content-Type` header. + +For MDX pages without layouts, `@astrojs/mdx` will automatically add the `` tag to the page by default. This reduces the boilerplate needed to write with non-ASCII characters. If your MDX pages have a layout, the layout component should include the `` tag. + +If you require `charset=utf-8` to render your page correctly, make sure that your layout components have the `` tag added. diff --git a/.changeset/strong-months-grab.md b/.changeset/strong-months-grab.md new file mode 100644 index 000000000000..193031f185c9 --- /dev/null +++ b/.changeset/strong-months-grab.md @@ -0,0 +1,11 @@ +--- +'astro': major +--- + +Updates the automatic `charset=utf-8` behavior for Markdown pages, where instead of responding with `charset=utf-8` in the `Content-Type` header, Astro will now automatically add the `` tag instead. + +This behaviour only applies to Markdown pages (`.md` or similar Markdown files located within `src/pages/`) that do not use Astro's special `layout` frontmatter property. It matches the rendering behaviour of other non-content pages, and retains the minimal boilerplate needed to write with non-ASCII characters when adding individual Markdown pages to your site. + +If your Markdown pages use the `layout` frontmatter property, then HTML encoding will be handled by the designated layout component instead, and the `` tag will not be added to your page by default. + +If you require `charset=utf-8` to render your page correctly, make sure that your layout components contain the `` tag. You may need to add this if you have not already done so. diff --git a/packages/astro/src/core/preview/vite-plugin-astro-preview.ts b/packages/astro/src/core/preview/vite-plugin-astro-preview.ts index 992f82f143dc..a8043192dfc9 100644 --- a/packages/astro/src/core/preview/vite-plugin-astro-preview.ts +++ b/packages/astro/src/core/preview/vite-plugin-astro-preview.ts @@ -16,7 +16,7 @@ export function vitePluginAstroPreview(settings: AstroSettings): Plugin { const errorPagePath = fileURLToPath(outDir + '/404.html'); if (fs.existsSync(errorPagePath)) { res.statusCode = 404; - res.setHeader('Content-Type', 'text/html;charset=utf-8'); + res.setHeader('Content-Type', 'text/html'); res.end(fs.readFileSync(errorPagePath)); } else { res.statusCode = 404; diff --git a/packages/astro/src/core/routing/astro-designed-error-pages.ts b/packages/astro/src/core/routing/astro-designed-error-pages.ts index 2c1c1f77c5a3..671221b5d68a 100644 --- a/packages/astro/src/core/routing/astro-designed-error-pages.ts +++ b/packages/astro/src/core/routing/astro-designed-error-pages.ts @@ -46,7 +46,7 @@ async function default404Page({ pathname }: { pathname: string }) { tabTitle: '404: Not Found', pathname, }), - { status: 404, headers: { 'Content-Type': 'text/html; charset=utf-8' } }, + { status: 404, headers: { 'Content-Type': 'text/html' } }, ); } // mark the function as an AstroComponentFactory for the rendering internals diff --git a/packages/astro/src/runtime/server/render/page.ts b/packages/astro/src/runtime/server/render/page.ts index 0e0bcf295ad8..5ac05f741347 100644 --- a/packages/astro/src/runtime/server/render/page.ts +++ b/packages/astro/src/runtime/server/render/page.ts @@ -35,7 +35,7 @@ export async function renderPage( return new Response(bytes, { headers: new Headers([ - ['Content-Type', 'text/html; charset=utf-8'], + ['Content-Type', 'text/html'], ['Content-Length', bytes.byteLength.toString()], ]), }); @@ -80,11 +80,6 @@ export async function renderPage( body = encoder.encode(body); headers.set('Content-Length', body.byteLength.toString()); } - // TODO: Revisit if user should manually set charset by themselves in Astro 4 - // This code preserves the existing behaviour for markdown pages since Astro 2 - if (route?.component.endsWith('.md')) { - headers.set('Content-Type', 'text/html; charset=utf-8'); - } let status = init.status; // Custom 404.astro and 500.astro are particular routes that must return a fixed status code if (route?.route === '/404') { diff --git a/packages/astro/src/vite-plugin-astro-server/response.ts b/packages/astro/src/vite-plugin-astro-server/response.ts index a6257508ee6d..ef3c8247aac9 100644 --- a/packages/astro/src/vite-plugin-astro-server/response.ts +++ b/packages/astro/src/vite-plugin-astro-server/response.ts @@ -46,7 +46,7 @@ export async function handle500Response( export function writeHtmlResponse(res: http.ServerResponse, statusCode: number, html: string) { res.writeHead(statusCode, { - 'Content-Type': 'text/html; charset=utf-8', + 'Content-Type': 'text/html', 'Content-Length': Buffer.byteLength(html, 'utf-8'), }); res.write(html); diff --git a/packages/astro/src/vite-plugin-markdown/index.ts b/packages/astro/src/vite-plugin-markdown/index.ts index b728ffb7048d..9b7c80330a05 100644 --- a/packages/astro/src/vite-plugin-markdown/index.ts +++ b/packages/astro/src/vite-plugin-markdown/index.ts @@ -10,7 +10,7 @@ import { normalizePath } from 'vite'; import { safeParseFrontmatter } from '../content/utils.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; import type { Logger } from '../core/logger/core.js'; -import { isMarkdownFile } from '../core/util.js'; +import { isMarkdownFile, isPage } from '../core/util.js'; import { shorthash } from '../runtime/server/shorthash.js'; import type { AstroSettings } from '../types/astro.js'; import { createDefaultAstroMetadata } from '../vite-plugin-astro/metadata.js'; @@ -77,6 +77,10 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug let html = renderResult.code; const { headings, imagePaths: rawImagePaths, frontmatter } = renderResult.metadata; + // Add default charset for markdown pages + const isMarkdownPage = isPage(fileURL, settings); + const charset = isMarkdownPage ? '' : ''; + // Resolve all the extracted images from the content const imagePaths: MarkdownImagePath[] = []; for (const imagePath of rawImagePaths) { @@ -141,7 +145,7 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug }, { 'default': () => render\`\${unescapeHTML(html())}\` })}\`;` - : `render\`\${maybeRenderHead(result)}\${unescapeHTML(html())}\`;` + : `render\`${charset}\${maybeRenderHead(result)}\${unescapeHTML(html())}\`;` } }); export default Content; diff --git a/packages/astro/test/astro-basic.test.js b/packages/astro/test/astro-basic.test.js index 0fc87dd6068e..144eec810125 100644 --- a/packages/astro/test/astro-basic.test.js +++ b/packages/astro/test/astro-basic.test.js @@ -115,12 +115,7 @@ describe('Astro basic build', () => { const html = await fixture.readFile('/chinese-encoding-md/index.html'); const $ = cheerio.load(html); assert.equal($('h1').text(), '我的第一篇博客文章'); - }); - - it('renders MDX in utf-8 by default', async () => { - const html = await fixture.readFile('/chinese-encoding-mdx/index.html'); - const $ = cheerio.load(html); - assert.equal($('h1').text(), '我的第一篇博客文章'); + assert.match(html, / { @@ -207,22 +202,8 @@ describe('Astro basic development', () => { const html = await res.text(); const $ = cheerio.load(html); assert.equal($('h1').text(), '我的第一篇博客文章'); - const isUtf8 = - res.headers.get('content-type').includes('charset=utf-8') || - html.includes(''); - assert.ok(isUtf8); - }); - - it('Renders MDX in utf-8 by default', async () => { - const res = await fixture.fetch('/chinese-encoding-mdx'); - assert.equal(res.status, 200); - const html = await res.text(); - const $ = cheerio.load(html); - assert.equal($('h1').text(), '我的第一篇博客文章'); - const isUtf8 = - res.headers.get('content-type').includes('charset=utf-8') || - html.includes(''); - assert.ok(isUtf8); + assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/); + assert.match(html, / { diff --git a/packages/integrations/mdx/src/index.ts b/packages/integrations/mdx/src/index.ts index dcb13bc62f70..fb6766e5f721 100644 --- a/packages/integrations/mdx/src/index.ts +++ b/packages/integrations/mdx/src/index.ts @@ -13,7 +13,7 @@ import type { PluggableList } from 'unified'; import type { OptimizeOptions } from './rehype-optimize-static.js'; import { ignoreStringPlugins, safeParseFrontmatter } from './utils.js'; import { vitePluginMdxPostprocess } from './vite-plugin-mdx-postprocess.js'; -import { vitePluginMdx } from './vite-plugin-mdx.js'; +import { type VitePluginMdxOptions, vitePluginMdx } from './vite-plugin-mdx.js'; export type MdxOptions = Omit & { extendMarkdownConfig: boolean; @@ -43,7 +43,7 @@ export function getContainerRenderer(): ContainerRenderer { export default function mdx(partialMdxOptions: Partial = {}): AstroIntegration { // @ts-expect-error Temporarily assign an empty object here, which will be re-assigned by the // `astro:config:done` hook later. This is so that `vitePluginMdx` can get hold of a reference earlier. - let mdxOptions: MdxOptions = {}; + let vitePluginMdxOptions: VitePluginMdxOptions = {}; return { name: '@astrojs/mdx', @@ -79,7 +79,7 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI updateConfig({ vite: { - plugins: [vitePluginMdx(mdxOptions), vitePluginMdxPostprocess(config)], + plugins: [vitePluginMdx(vitePluginMdxOptions), vitePluginMdxPostprocess(config)], }, }); }, @@ -98,10 +98,13 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI }); // Mutate `mdxOptions` so that `vitePluginMdx` can reference the actual options - Object.assign(mdxOptions, resolvedMdxOptions); + Object.assign(vitePluginMdxOptions, { + mdxOptions: resolvedMdxOptions, + srcDir: config.srcDir, + }); // @ts-expect-error After we assign, we don't need to reference `mdxOptions` in this context anymore. // Re-assign it so that the garbage can be collected later. - mdxOptions = {}; + vitePluginMdxOptions = {}; }, }, }; diff --git a/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts b/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts index cc1f4d141bc4..5880c30b318b 100644 --- a/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts +++ b/packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts @@ -1,9 +1,23 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { isFrontmatterValid } from '@astrojs/markdown-remark'; +import type { Root, RootContent } from 'hast'; import type { VFile } from 'vfile'; import { jsToTreeNode } from './utils.js'; +// Passed metadata to help determine adding charset utf8 by default +declare module 'vfile' { + interface DataMap { + applyFrontmatterExport?: { + srcDir?: URL; + }; + } +} + +const exportConstPartialTrueRe = /export\s+const\s+partial\s*=\s*true/; + export function rehypeApplyFrontmatterExport() { - return function (tree: any, vfile: VFile) { + return function (tree: Root, vfile: VFile) { const frontmatter = vfile.data.astro?.frontmatter; if (!frontmatter || !isFrontmatterValid(frontmatter)) throw new Error( @@ -11,11 +25,11 @@ export function rehypeApplyFrontmatterExport() { // TODO: find way to import error data from core '[MDX] A remark or rehype plugin attempted to inject invalid frontmatter. Ensure "astro.frontmatter" is set to a valid JSON object that is not `null` or `undefined`.', ); - const exportNodes = [ + const extraChildren: RootContent[] = [ jsToTreeNode(`export const frontmatter = ${JSON.stringify(frontmatter)};`), ]; if (frontmatter.layout) { - exportNodes.unshift( + extraChildren.unshift( jsToTreeNode( // NOTE: Use `__astro_*` import names to prevent conflicts with user code /** @see 'vite-plugin-markdown' for layout props reference */ @@ -39,7 +53,61 @@ export default function ({ children }) { };`, ), ); + } else if (shouldAddCharset(tree, vfile)) { + extraChildren.unshift({ + type: 'mdxJsxFlowElement', + name: 'meta', + attributes: [ + { + type: 'mdxJsxAttribute', + name: 'charset', + value: 'utf-8', + }, + ], + children: [], + }); } - tree.children = exportNodes.concat(tree.children); + tree.children = extraChildren.concat(tree.children); }; } + +/** + * If this is a page (e.g. in src/pages), has no layout frontmatter (handled before calling this function), + * has no leading component that looks like a wrapping layout, and `partial` isn't set to true, we default to + * adding charset=utf-8 like markdown so that users don't have to worry about it for MDX pages without layouts. + */ +function shouldAddCharset(tree: Root, vfile: VFile) { + const srcDirUrl = vfile.data.applyFrontmatterExport?.srcDir; + if (!srcDirUrl) return false; + + const hasConstPartialTrue = tree.children.some( + (node) => node.type === 'mdxjsEsm' && exportConstPartialTrueRe.test(node.value), + ); + if (hasConstPartialTrue) return false; + + // NOTE: the pages directory is a non-configurable Astro behaviour + const pagesDir = path.join(fileURLToPath(srcDirUrl), 'pages').replace(/\\/g, '/'); + // `vfile.path` comes from Vite, which is a normalized path (no backslashes) + const filePath = vfile.path; + if (!filePath.startsWith(pagesDir)) return false; + + const hasLeadingUnderscoreInPath = filePath + .slice(pagesDir.length) + .replace(/\\/g, '/') + .split('/') + .some((part) => part.startsWith('_')); + if (hasLeadingUnderscoreInPath) return false; + + // Bail if the first content found is a wrapping layout component + for (const child of tree.children) { + if (child.type === 'element') break; + if (child.type === 'mdxJsxFlowElement') { + // If is fragment or lowercase tag name (html tags), skip and assume there's no layout + if (child.name == null) break; + if (child.name[0] === child.name[0].toLowerCase()) break; + return false; + } + } + + return true; +} diff --git a/packages/integrations/mdx/src/vite-plugin-mdx.ts b/packages/integrations/mdx/src/vite-plugin-mdx.ts index eea530c1c31b..869c65d26502 100644 --- a/packages/integrations/mdx/src/vite-plugin-mdx.ts +++ b/packages/integrations/mdx/src/vite-plugin-mdx.ts @@ -6,7 +6,13 @@ import type { MdxOptions } from './index.js'; import { createMdxProcessor } from './plugins.js'; import { safeParseFrontmatter } from './utils.js'; -export function vitePluginMdx(mdxOptions: MdxOptions): Plugin { +export interface VitePluginMdxOptions { + mdxOptions: MdxOptions; + srcDir: URL; +} + +// NOTE: Do not destructure `opts` as we're assigning a reference that will be mutated later +export function vitePluginMdx(opts: VitePluginMdxOptions): Plugin { let processor: ReturnType | undefined; let sourcemapEnabled: boolean; @@ -47,12 +53,15 @@ export function vitePluginMdx(mdxOptions: MdxOptions): Plugin { astro: { frontmatter, }, + applyFrontmatterExport: { + srcDir: opts.srcDir, + }, }, }); // Lazily initialize the MDX processor if (!processor) { - processor = createMdxProcessor(mdxOptions, { sourcemap: sourcemapEnabled }); + processor = createMdxProcessor(opts.mdxOptions, { sourcemap: sourcemapEnabled }); } try { diff --git a/packages/integrations/mdx/test/fixtures/mdx-page/src/layouts/EncodingLayout.astro b/packages/integrations/mdx/test/fixtures/mdx-page/src/layouts/EncodingLayout.astro new file mode 100644 index 000000000000..13e0e91ed627 --- /dev/null +++ b/packages/integrations/mdx/test/fixtures/mdx-page/src/layouts/EncodingLayout.astro @@ -0,0 +1 @@ + diff --git a/packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-frontmatter.mdx b/packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-frontmatter.mdx new file mode 100644 index 000000000000..471827de01a9 --- /dev/null +++ b/packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-frontmatter.mdx @@ -0,0 +1,7 @@ +--- +layout: ../layouts/EncodingLayout.astro +--- + +# 我的第一篇博客文章 + +发表于：2022-07-01 diff --git a/packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-manual.mdx b/packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-manual.mdx new file mode 100644 index 000000000000..1c8c78630788 --- /dev/null +++ b/packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-manual.mdx @@ -0,0 +1,12 @@ +import EncodingLayout from '../layouts/EncodingLayout.astro' + +{/* Ensure random stuff preceding the wrapper layout is ignored when detecting a wrapper layout */} +export const foo = {} + + + +# 我的第一篇博客文章 + +发表于：2022-07-01 + + diff --git a/packages/astro/test/fixtures/astro-basic/src/pages/chinese-encoding-mdx.mdx b/packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding.mdx similarity index 100% rename from packages/astro/test/fixtures/astro-basic/src/pages/chinese-encoding-mdx.mdx rename to packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding.mdx diff --git a/packages/integrations/mdx/test/mdx-page.test.js b/packages/integrations/mdx/test/mdx-page.test.js index 7948de65312a..b58781efca21 100644 --- a/packages/integrations/mdx/test/mdx-page.test.js +++ b/packages/integrations/mdx/test/mdx-page.test.js @@ -1,5 +1,6 @@ import * as assert from 'node:assert/strict'; import { after, before, describe, it } from 'node:test'; +import * as cheerio from 'cheerio'; import { parseHTML } from 'linkedom'; import { loadFixture } from '../../../astro/test/test-utils.js'; @@ -36,6 +37,23 @@ describe('MDX Page', () => { assert.notEqual(stylesheet, null); }); + + it('Renders MDX in utf-8 by default', async () => { + const html = await fixture.readFile('/chinese-encoding/index.html'); + const $ = cheerio.load(html); + assert.equal($('h1').text(), '我的第一篇博客文章'); + assert.match(html, / { + const html = await fixture.readFile('/chinese-encoding-layout-frontmatter/index.html'); + assert.doesNotMatch(html, / { + const html = await fixture.readFile('/chinese-encoding-layout-manual/index.html'); + assert.doesNotMatch(html, / { @@ -61,5 +79,31 @@ describe('MDX Page', () => { assert.equal(h1.textContent, 'Hello page!'); }); + + it('Renders MDX in utf-8 by default', async () => { + const res = await fixture.fetch('/chinese-encoding/'); + assert.equal(res.status, 200); + const html = await res.text(); + const $ = cheerio.load(html); + assert.equal($('h1').text(), '我的第一篇博客文章'); + assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/); + assert.match(html, / { + const res = await fixture.fetch('/chinese-encoding-layout-frontmatter/'); + assert.equal(res.status, 200); + const html = await res.text(); + assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/); + assert.doesNotMatch(html, / { + const res = await fixture.fetch('/chinese-encoding-layout-manual/'); + assert.equal(res.status, 200); + const html = await res.text(); + assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/); + assert.doesNotMatch(html, /