Data collections and references

.changeset/early-eyes-bow.md

@@ -0,0 +1,6 @@
+---
+'astro': minor
+'@astrojs/markdoc': minor
+---
+
+Content collections now support data formats including JSON and YAML. You can also create relationships, or references, between collections to pull information from one collection entry into another. Learn more on our [updated Content Collections docs](https://docs.astro.build/en/guides/content-collections/).

package.json

@@ -18,6 +18,8 @@
     "format:imports": "organize-imports-cli ./packages/*/tsconfig.json ./packages/*/*/tsconfig.json",
     "test": "turbo run test --concurrency=1 --filter=astro --filter=create-astro --filter=\"@astrojs/*\"",
     "test:match": "cd packages/astro && pnpm run test:match",
+    "test:unit": "cd packages/astro && pnpm run test:unit",
+    "test:unit:match": "cd packages/astro && pnpm run test:unit:match",
     "test:smoke": "pnpm test:smoke:example && pnpm test:smoke:docs",
     "test:smoke:example": "turbo run build --concurrency=100% --filter=\"@example/*\"",
     "test:smoke:docs": "turbo run build --filter=docs",

packages/astro/package.json

@@ -144,6 +144,7 @@
     "github-slugger": "^2.0.0",
     "gray-matter": "^4.0.3",
     "html-escaper": "^3.0.3",
+    "js-yaml": "^4.1.0",
     "kleur": "^4.1.4",
     "magic-string": "^0.27.0",
     "mime": "^3.0.0",
@@ -181,6 +182,7 @@
     "@types/estree": "^0.0.51",
     "@types/hast": "^2.3.4",
     "@types/html-escaper": "^3.0.0",
+    "@types/js-yaml": "^4.0.5",
     "@types/mime": "^2.0.3",
     "@types/mocha": "^9.1.1",
     "@types/prettier": "^2.6.3",

packages/astro/src/@types/astro.ts

@@ -1249,12 +1249,22 @@ export type ContentEntryModule = {
 	};
 };
 
+export type DataEntryModule = {
+	id: string;
+	collection: string;
+	data: Record<string, unknown>;
+	_internal: {
+		rawData: string;
+		filePath: string;
+	};
+};
+
 export interface ContentEntryType {
 	extensions: string[];
 	getEntryInfo(params: {
 		fileUrl: URL;
 		contents: string;
-	}): GetEntryInfoReturnType | Promise<GetEntryInfoReturnType>;
+	}): GetContentEntryInfoReturnType | Promise<GetContentEntryInfoReturnType>;
 	getRenderModule?(
 		this: rollup.PluginContext,
 		params: {
@@ -1265,7 +1275,7 @@ export interface ContentEntryType {
 	contentModuleTypes?: string;
 }
 
-type GetEntryInfoReturnType = {
+type GetContentEntryInfoReturnType = {
 	data: Record<string, unknown>;
 	/**
 	 * Used for error hints to point to correct line and location
@@ -1277,12 +1287,23 @@ type GetEntryInfoReturnType = {
 	slug: string;
 };
 
+export interface DataEntryType {
+	extensions: string[];
+	getEntryInfo(params: {
+		fileUrl: URL;
+		contents: string;
+	}): GetDataEntryInfoReturnType | Promise<GetDataEntryInfoReturnType>;
+}
+
+export type GetDataEntryInfoReturnType = { data: Record<string, unknown>; rawData?: string };
+
 export interface AstroSettings {
 	config: AstroConfig;
 	adapter: AstroAdapter | undefined;
 	injectedRoutes: InjectedRoute[];
 	pageExtensions: string[];
 	contentEntryTypes: ContentEntryType[];
+	dataEntryTypes: DataEntryType[];
 	renderers: AstroRenderer[];
 	scripts: {
 		stage: InjectedScriptStage;

packages/astro/src/content/consts.ts

@@ -1,5 +1,8 @@
 export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets';
-export const CONTENT_FLAG = 'astroContent';
+export const CONTENT_FLAG = 'astroContentCollectionEntry';
+export const DATA_FLAG = 'astroDataCollectionEntry';
+export const CONTENT_FLAGS = [CONTENT_FLAG, DATA_FLAG, PROPAGATED_ASSET_FLAG] as const;
+
 export const VIRTUAL_MODULE_ID = 'astro:content';
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
 export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@';

packages/astro/src/content/runtime-assets.ts

@@ -4,7 +4,7 @@ import type { AstroSettings } from '../@types/astro.js';
 import { emitESMImage } from '../assets/index.js';
 
 export function createImage(
-	settings: AstroSettings,
+	settings: Pick<AstroSettings, 'config'>,
 	pluginContext: PluginContext,
 	entryFilePath: string
 ) {

packages/astro/src/content/runtime.ts

@@ -1,6 +1,6 @@
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
 import { prependForwardSlash } from '../core/path.js';
-
+import { ZodIssueCode, string as zodString, type z } from 'zod';
 import {
 	createComponent,
 	createHeadAndContent,
@@ -9,7 +9,10 @@ import {
 	renderTemplate,
 	renderUniqueStylesheet,
 	unescapeHTML,
+	type AstroComponentFactory,
 } from '../runtime/server/index.js';
+import type { ContentLookupMap } from './utils.js';
+import type { MarkdownHeading } from '@astrojs/markdown-remark';
 
 type LazyImport = () => Promise<any>;
 type GlobResult = Record<string, LazyImport>;
@@ -37,14 +40,31 @@ export function createCollectionToGlobResultMap({
 
 const cacheEntriesByCollection = new Map<string, any[]>();
 export function createGetCollection({
-	collectionToEntryMap,
+	contentCollectionToEntryMap,
+	dataCollectionToEntryMap,
 	getRenderEntryImport,
 }: {
-	collectionToEntryMap: CollectionToEntryMap;
+	contentCollectionToEntryMap: CollectionToEntryMap;
+	dataCollectionToEntryMap: CollectionToEntryMap;
 	getRenderEntryImport: GetEntryImport;
 }) {
 	return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
-		const lazyImports = Object.values(collectionToEntryMap[collection] ?? {});
+		let type: 'content' | 'data';
+		if (collection in contentCollectionToEntryMap) {
+			type = 'content';
+		} else if (collection in dataCollectionToEntryMap) {
+			type = 'data';
+		} else {
+			throw new AstroError({
+				...AstroErrorData.CollectionDoesNotExistError,
+				message: AstroErrorData.CollectionDoesNotExistError.message(collection),
+			});
+		}
+		const lazyImports = Object.values(
+			type === 'content'
+				? contentCollectionToEntryMap[collection]
+				: dataCollectionToEntryMap[collection]
+		);
 		let entries: any[] = [];
 		// Cache `getCollection()` calls in production only
 		// prevents stale cache in development
@@ -54,20 +74,26 @@ export function createGetCollection({
 			entries = await Promise.all(
 				lazyImports.map(async (lazyImport) => {
 					const entry = await lazyImport();
-					return {
-						id: entry.id,
-						slug: entry.slug,
-						body: entry.body,
-						collection: entry.collection,
-						data: entry.data,
-						async render() {
-							return render({
+					return type === 'content'
+						? {
+								id: entry.id,
+								slug: entry.slug,
+								body: entry.body,
 								collection: entry.collection,
+								data: entry.data,
+								async render() {
+									return render({
+										collection: entry.collection,
+										id: entry.id,
+										renderEntryImport: await getRenderEntryImport(collection, entry.slug),
+									});
+								},
+						  }
+						: {
 								id: entry.id,
-								renderEntryImport: await getRenderEntryImport(collection, entry.slug),
-							});
-						},
-					};
+								collection: entry.collection,
+								data: entry.data,
+						  };
 				})
 			);
 			cacheEntriesByCollection.set(collection, entries);
@@ -110,6 +136,121 @@ export function createGetEntryBySlug({
 	};
 }
 
+export function createGetDataEntryById({
+	dataCollectionToEntryMap,
+}: {
+	dataCollectionToEntryMap: CollectionToEntryMap;
+}) {
+	return async function getDataEntryById(collection: string, id: string) {
+		const lazyImport =
+			dataCollectionToEntryMap[collection]?.[/*TODO: filePathToIdMap*/ id + '.json'];
+
+		// TODO: AstroError
+		if (!lazyImport) throw new Error(`Entry ${collection} → ${id} was not found.`);
+		const entry = await lazyImport();
+
+		return {
+			id: entry.id,
+			collection: entry.collection,
+			data: entry.data,
+		};
+	};
+}
+
+type ContentEntryResult = {
+	id: string;
+	slug: string;
+	body: string;
+	collection: string;
+	data: Record<string, any>;
+	render(): Promise<RenderResult>;
+};
+
+type DataEntryResult = {
+	id: string;
+	collection: string;
+	data: Record<string, any>;
+};
+
+type EntryLookupObject = { collection: string; id: string } | { collection: string; slug: string };
+
+export function createGetEntry({
+	getEntryImport,
+	getRenderEntryImport,
+}: {
+	getEntryImport: GetEntryImport;
+	getRenderEntryImport: GetEntryImport;
+}) {
+	return async function getEntry(
+		// Can either pass collection and identifier as 2 positional args,
+		// Or pass a single object with the collection and identifier as properties.
+		// This means the first positional arg can have different shapes.
+		collectionOrLookupObject: string | EntryLookupObject,
+		_lookupId?: string
+	): Promise<ContentEntryResult | DataEntryResult | undefined> {
+		let collection: string, lookupId: string;
+		if (typeof collectionOrLookupObject === 'string') {
+			collection = collectionOrLookupObject;
+			if (!_lookupId)
+				throw new AstroError({
+					...AstroErrorData.UnknownContentCollectionError,
+					message: '`getEntry()` requires an entry identifier as the second argument.',
+				});
+			lookupId = _lookupId;
+		} else {
+			collection = collectionOrLookupObject.collection;
+			// Identifier could be `slug` for content entries, or `id` for data entries
+			lookupId =
+				'id' in collectionOrLookupObject
+					? collectionOrLookupObject.id
+					: collectionOrLookupObject.slug;
+		}
+
+		const entryImport = await getEntryImport(collection, lookupId);
+		if (typeof entryImport !== 'function') return undefined;
+
+		const entry = await entryImport();
+
+		if (entry._internal.type === 'content') {
+			return {
+				id: entry.id,
+				slug: entry.slug,
+				body: entry.body,
+				collection: entry.collection,
+				data: entry.data,
+				async render() {
+					return render({
+						collection: entry.collection,
+						id: entry.id,
+						renderEntryImport: await getRenderEntryImport(collection, lookupId),
+					});
+				},
+			};
+		} else if (entry._internal.type === 'data') {
+			return {
+				id: entry.id,
+				collection: entry.collection,
+				data: entry.data,
+			};
+		}
+		return undefined;
+	};
+}
+
+export function createGetEntries(getEntry: ReturnType<typeof createGetEntry>) {
+	return async function getEntries(
+		entries: { collection: string; id: string }[] | { collection: string; slug: string }[]
+	) {
+		return Promise.all(entries.map((e) => getEntry(e)));
+	};
+}
+
+type RenderResult = {
+	Content: AstroComponentFactory;
+	headings: MarkdownHeading[];
+	remarkPluginFrontmatter: Record<string, any>;
+};
+
 async function render({
 	collection,
 	id,
@@ -118,7 +259,7 @@ async function render({
 	collection: string;
 	id: string;
 	renderEntryImport?: LazyImport;
-}) {
+}): Promise<RenderResult> {
 	const UnexpectedRenderError = new AstroError({
 		...AstroErrorData.UnknownContentCollectionError,
 		message: `Unexpected error while rendering ${String(collection)} → ${String(id)}.`,
@@ -186,3 +327,38 @@ async function render({
 		remarkPluginFrontmatter: mod.frontmatter ?? {},
 	};
 }
+
+export function createReference({ lookupMap }: { lookupMap: ContentLookupMap }) {
+	return function reference(collection: string) {
+		return zodString().transform((lookupId: string, ctx) => {
+			const flattenedErrorPath = ctx.path.join('.');
+			if (!lookupMap[collection]) {
+				ctx.addIssue({
+					code: ZodIssueCode.custom,
+					message: `**${flattenedErrorPath}:** Reference to ${collection} invalid. Collection does not exist or is empty.`,
+				});
+				return;
+			}
+
+			const { type, entries } = lookupMap[collection];
+			const entry = entries[lookupId];
+
+			if (!entry) {
+				ctx.addIssue({
+					code: ZodIssueCode.custom,
+					message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Expected ${Object.keys(
+						entries
+					)
+						.map((c) => JSON.stringify(c))
+						.join(' | ')}. Received ${JSON.stringify(lookupId)}.`,
+				});
+				return;
+			}
+			// Content is still identified by slugs, so map to a `slug` key for consistency.
+			if (type === 'content') {
+				return { slug: lookupId, collection };
+			}
+			return { id: lookupId, collection };
+		});
+	};
+}