feat: Add TOML support to the built-in content loaders

.changeset/icy-keys-tickle.md

@@ -0,0 +1,28 @@
+---
+'astro': minor
+'@astrojs/markdown-remark': patch
+---
+
+Adds support for TOML files to Astro's built-in `glob()` and `file()` content loaders.
+
+In Astro 5.2, Astro added support for using TOML frontmatter in Markdown files instead of YAML. However, if you wanted to use TOML files as local content collection entries themselves, you needed to write your own loader.
+
+Astro 5.12 now directly supports loading data from TOML files in content collections in both the `glob()` and the `file()` loaders.
+
+If you had added your own TOML content parser for the `file()` loader,  you can now remove it as this functionality is now included:
+
+```diff
+// src/content.config.ts
+import { defineCollection } from "astro:content";
+import { file } from "astro/loaders";
+- import { parse as parseToml } from "toml";
+const dogs = defineCollection({
+-  loader: file("src/data/dogs.toml", { parser: (text) => parseToml(text) }),
++ loader: file("src/data/dogs.toml")
+  schema: /* ... */
+})
+```
+
+Note that TOML does not support top-level arrays. Instead, the `file()` loader considers each top-level table to be an independent entry. The table header is populated in the `id` field of the entry object. 
+
+See Astro's [content collections guide](https://docs.astro.build/en/guides/content-collections/#built-in-loaders) for more information on using the built-in content loaders.

packages/astro/package.json

@@ -159,6 +159,7 @@
     "rehype": "^13.0.2",
     "semver": "^7.7.1",
     "shiki": "^3.2.1",
+    "smol-toml": "^1.3.4",
     "tinyexec": "^0.3.2",
     "tinyglobby": "^0.2.12",
     "tsconfck": "^3.1.5",

packages/astro/src/content/loaders/file.ts

@@ -1,6 +1,7 @@
 import { existsSync, promises as fs } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import yaml from 'js-yaml';
+import toml from 'smol-toml';
 import { FileGlobNotSupported, FileParserNotFound } from '../../core/errors/errors-data.js';
 import { AstroError } from '../../core/errors/index.js';
 import { posixRelative } from '../utils.js';
@@ -36,6 +37,8 @@ export function file(fileName: string, options?: FileOptions): Loader {
 			yaml.load(text, {
 				filename: fileName,
 			});
+	} else if (ext === 'toml') {
+		parse = toml.parse;
 	}
 	if (options?.parser) parse = options.parser;
 

packages/astro/src/core/config/settings.ts

@@ -1,6 +1,7 @@
 import path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import yaml from 'js-yaml';
+import toml from 'smol-toml';
 import { getContentPaths } from '../../content/index.js';
 import createPreferences from '../../preferences/index.js';
 import type { AstroSettings } from '../../types/astro.js';
@@ -9,7 +10,12 @@ import { markdownContentEntryType } from '../../vite-plugin-markdown/content-ent
 import { getDefaultClientDirectives } from '../client-directive/index.js';
 import { SUPPORTED_MARKDOWN_FILE_EXTENSIONS } from './../constants.js';
 import { AstroError, AstroErrorData } from '../errors/index.js';
-import { formatYAMLException, isYAMLException } from '../errors/utils.js';
+import {
+	formatTOMLError,
+	formatYAMLException,
+	isTOMLError,
+	isYAMLException,
+} from '../errors/utils.js';
 import { AstroTimer } from './timer.js';
 import { loadTSConfig } from './tsconfig.js';
 
@@ -100,6 +106,38 @@ export function createBaseSettings(config: AstroConfig): AstroSettings {
 					}
 				},
 			},
+			{
+				extensions: ['.toml'],
+				getEntryInfo({ contents, fileUrl }) {
+					try {
+						const data = toml.parse(contents);
+						const rawData = contents;
+
+						return { data, rawData };
+					} catch (e) {
+						const pathRelToContentDir = path.relative(
+							fileURLToPath(contentDir),
+							fileURLToPath(fileUrl),
+						);
+						const formattedError = isTOMLError(e)
+							? formatTOMLError(e)
+							: new Error('contains invalid TOML.');
+
+						throw new AstroError({
+							...AstroErrorData.DataCollectionEntryParseError,
+							message: AstroErrorData.DataCollectionEntryParseError.message(
+								pathRelToContentDir,
+								formattedError.message,
+							),
+							stack: formattedError.stack,
+							location:
+								'loc' in formattedError
+									? { file: fileUrl.pathname, ...formattedError.loc }
+									: { file: fileUrl.pathname },
+						});
+					}
+				},
+			},
 		],
 		renderers: [],
 		scripts: [],

packages/astro/src/core/errors/errors-data.ts

@@ -1805,15 +1805,15 @@ export const ContentCollectionTypeMismatchError = {
  * @docs
  * @message `COLLECTION_ENTRY_NAME` failed to parse.
  * @description
- * Collection entries of `type: 'data'` must return an object with valid JSON (for `.json` entries) or YAML (for `.yaml` entries).
+ * Collection entries of `type: 'data'` must return an object with valid JSON (for `.json` entries), YAML (for `.yaml` entries) or TOML (for `.toml` entries).'
  */
 export const DataCollectionEntryParseError = {
 	name: 'DataCollectionEntryParseError',
 	title: 'Data collection entry failed to parse.',
 	message(entryId: string, errorMessage: string) {
 		return `**${entryId}** failed to parse: ${errorMessage}`;
 	},
-	hint: 'Ensure your data entry is an object with valid JSON (for `.json` entries) or YAML (for `.yaml` entries).',
+	hint: 'Ensure your data entry is an object with valid JSON (for `.json` entries), YAML (for `.yaml` entries) or TOML (for `.toml` entries).',
 } satisfies ErrorData;
 /**
  * @docs
@@ -1855,7 +1855,7 @@ export const UnsupportedConfigTransformError = {
  * @see
  *  - [Passing a `parser` to the `file` loader](https://docs.astro.build/en/guides/content-collections/#parser-function)
  * @description
- * The `file` loader can’t determine which parser to use. Please provide a custom parser (e.g. `toml.parse` or `csv-parse`) to create a collection from your file type.
+ * The `file` loader can’t determine which parser to use. Please provide a custom parser (e.g. `csv-parse`) to create a collection from your file type.
  */
 export const FileParserNotFound = {
 	name: 'FileParserNotFound',

packages/astro/src/core/errors/utils.ts

@@ -1,4 +1,5 @@
 import type { YAMLException } from 'js-yaml';
+import type { TomlError } from 'smol-toml';
 import type { ErrorPayload as ViteErrorPayload } from 'vite';
 import type { SSRError } from '../../types/public/internal.js';
 
@@ -86,6 +87,21 @@ export function formatYAMLException(e: YAMLException): ViteErrorPayload['err'] {
 	};
 }
 
+export function isTOMLError(err: unknown): err is TomlError {
+	return err instanceof Error && err.name === 'TomlError';
+}
+
+/** Format TOML exceptions as Vite errors */
+export function formatTOMLError(e: TomlError): ViteErrorPayload['err'] {
+	return {
+		name: e.name,
+		id: e.name,
+		loc: { line: e.line + 1, column: e.column },
+		message: e.message,
+		stack: e.stack ?? '',
+	};
+}
+
 /** Coalesce any throw variable to an Error instance. */
 export function createSafeError(err: any): Error {
 	if (err instanceof Error || (err?.name && err.message)) {

packages/astro/test/content-layer.test.js

@@ -137,7 +137,7 @@ describe('Content Layer', () => {
 			assert.ok(json.hasOwnProperty('tomlLoader'));
 			assert.ok(Array.isArray(json.tomlLoader));
 
-			const ids = json.tomlLoader.map((item) => item.data.id);
+			const ids = json.tomlLoader.map((item) => item.id);
 			assert.deepEqual(ids, [
 				'crown',
 				'nikes-on-my-feet',
@@ -150,6 +150,41 @@ describe('Content Layer', () => {
 			]);
 		});
 
+		it('Returns csv `file()` loader collection', async () => {
+			assert.ok(json.hasOwnProperty('csvLoader'));
+			assert.ok(Array.isArray(json.csvLoader));
+
+			const ids = json.csvLoader.map((item) => item.data.id);
+			assert.deepEqual(ids, [
+				'lavender',
+				'rose',
+				'sunflower',
+				'basil',
+				'thyme',
+				'sage',
+				'daisy',
+				'marigold',
+				'chamomile',
+				'fern',
+			]);
+		});
+
+		it('Returns yaml `glob()` loader collection', async () => {
+			assert.ok(json.hasOwnProperty('numbersYaml'));
+			assert.ok(Array.isArray(json.numbersYaml));
+
+			const titles = json.numbersYaml.map((item) => item.data.title).sort();
+			assert.deepEqual(titles, ['One', 'Three', 'Two']);
+		});
+
+		it('Returns toml `glob()` loader collection', async () => {
+			assert.ok(json.hasOwnProperty('numbersToml'));
+			assert.ok(Array.isArray(json.numbersToml));
+
+			const titles = json.numbersToml.map((item) => item.data.title).sort();
+			assert.deepEqual(titles, ['One', 'Three', 'Two']);
+		});
+
 		it('Returns nested json `file()` loader collection', async () => {
 			assert.ok(json.hasOwnProperty('nestedJsonLoader'));
 			assert.ok(Array.isArray(json.nestedJsonLoader));

packages/astro/test/fixtures/content-layer/package.json

@@ -4,7 +4,6 @@
   "private": true,
   "dependencies": {
     "astro": "workspace:*",
-    "@astrojs/mdx": "workspace:*",
-    "toml": "^3.0.0"
+    "@astrojs/mdx": "workspace:*"
   }
 }

packages/astro/test/fixtures/content-layer/src/content.config.ts

@@ -1,7 +1,6 @@
 import { defineCollection, z, reference } from 'astro:content';
 import { file, glob } from 'astro/loaders';
 import { loader } from './loaders/post-loader.js';
-import { parse as parseToml } from 'toml';
 import { readFile } from 'fs/promises';
 
 const blog = defineCollection({
@@ -130,6 +129,22 @@ const birds = defineCollection({
 	}),
 });
 
+const plants = defineCollection({
+	loader: file('src/data/plants.csv', {
+		parser: (text) => {
+			const [headers, ...rows] = text.trim().split('\n');
+			return rows.map(row => Object.fromEntries(
+			    headers.split(',').map((h, i) => [h, row.split(',')[i]])
+			));
+		},
+	}),
+	schema: z.object({
+		id: z.string(),
+		common_name: z.string(),
+		scientific_name: z.string(),
+		color: z.string(),
+	}),
+});
 
 const probes = defineCollection({
 	loader: glob({ pattern: ['*.md', '!voyager-*'], base: 'src/data/space-probes' }),
@@ -148,6 +163,14 @@ const numbers = defineCollection({
 	loader: glob({ pattern: 'src/data/glob-data/*', base: '.' }),
 });
 
+const numbersYaml = defineCollection({
+	loader: glob({ pattern: 'src/data/glob-yaml/*', base: '.' }),
+});
+
+const numbersToml = defineCollection({
+	loader: glob({ pattern: 'src/data/glob-toml/*', base: '.' }),
+});
+
 const notADirectory = defineCollection({
 	loader: glob({ pattern: '*', base: 'src/nonexistent' }),
 });
@@ -215,18 +238,16 @@ const increment = defineCollection({
 });
 
 const artists = defineCollection({
-        loader: file('src/data/music.toml', { parser: (text) => parseToml(text).artists }),
+        loader: file('src/data/artists.toml'),
         schema: z.object({
-                id: z.string(),
                 name: z.string(),
                 genre: z.string().array(),
         }),
 });
 
 const songs = defineCollection({
-        loader: file('src/data/music.toml', { parser: (text) => parseToml(text).songs }),
+        loader: file('src/data/songs.toml'),
         schema: z.object({
-                id: z.string(),
                 name: z.string(),
                 artists: z.array(reference('artists')),
         }),
@@ -238,7 +259,10 @@ export const collections = {
 	cats,
 	fish,
 	birds,
+	plants,
 	numbers,
+	numbersToml,
+	numbersYaml,
 	spacecraft,
 	increment,
 	images,

packages/astro/test/fixtures/content-layer/src/data/artists.toml

@@ -0,0 +1,39 @@
+[kendrick-lamar]
+name = "Kendrick Lamar"
+genre = ["Hip-Hop", "Rap"]
+
+[mac-miller]
+name = "Mac Miller"
+genre = ["Hip-Hop", "Rap"]
+
+[jid]
+name = "JID"
+genre = ["Hip-Hop", "Rap"]
+
+[yasiin-bey]
+name = "Yasiin Bey"
+genre = ["Hip-Hop", "Rap"]
+
+[kanye-west]
+name = "Kanye West"
+genre = ["Hip-Hop", "Rap"]
+
+[jay-z]
+name = "JAY-Z"
+genre = ["Hip-Hop", "Rap"]
+
+[j-ivy]
+name = "J. Ivy"
+genre = ["Spoken Word", "Rap"]
+
+[frank-ocean]
+name = "Frank Ocean"
+genre = ["R&B", "Hip-Hop"]
+
+[the-dream]
+name = "The-Dream"
+genre = ["R&B", "Hip-Hop"]
+
+[baby-keem]
+name = "Baby Keem"
+genre = ["Hip-Hop", "Rap"]

packages/astro/test/fixtures/content-layer/src/data/glob-toml/one.toml

@@ -0,0 +1 @@
+title = "One"

packages/astro/test/fixtures/content-layer/src/data/glob-toml/three.toml

@@ -0,0 +1 @@
+title = "Three"

packages/astro/test/fixtures/content-layer/src/data/glob-toml/two.toml

@@ -0,0 +1 @@
+title = "Two"

packages/astro/test/fixtures/content-layer/src/data/glob-yaml/one.yaml

@@ -0,0 +1 @@
+title: "One"

packages/astro/test/fixtures/content-layer/src/data/glob-yaml/three.yaml

@@ -0,0 +1 @@
+title: "Three"

packages/astro/test/fixtures/content-layer/src/data/glob-yaml/two.yaml

@@ -0,0 +1 @@
+title: "Two"