Data collections and references

.changeset/early-eyes-bow.md

@@ -0,0 +1,6 @@
+---
+'astro': minor
+'@astrojs/markdoc': patch
+---
+
+Introducing data collections and references. These offer a way to store JSON file as collection entries under the new `src/data/` directory. You can reference this data from existing content collections using "references." Learn more on our docs (TODO!)

packages/astro/package.json

@@ -143,6 +143,7 @@
     "github-slugger": "^2.0.0",
     "gray-matter": "^4.0.3",
     "html-escaper": "^3.0.3",
+    "js-yaml": "^4.1.0",
     "kleur": "^4.1.4",
     "magic-string": "^0.27.0",
     "mime": "^3.0.0",
@@ -180,6 +181,7 @@
     "@types/estree": "^0.0.51",
     "@types/hast": "^2.3.4",
     "@types/html-escaper": "^3.0.0",
+    "@types/js-yaml": "^4.0.5",
     "@types/mime": "^2.0.3",
     "@types/mocha": "^9.1.1",
     "@types/prettier": "^2.6.3",

packages/astro/src/@types/astro.ts

@@ -1167,12 +1167,22 @@ export type ContentEntryModule = {
 	};
 };
 
+export type DataEntryModule = {
+	id: string;
+	collection: string;
+	data: Record<string, unknown>;
+	_internal: {
+		rawData: string;
+		filePath: string;
+	};
+};
+
 export interface ContentEntryType {
 	extensions: string[];
 	getEntryInfo(params: {
 		fileUrl: URL;
 		contents: string;
-	}): GetEntryInfoReturnType | Promise<GetEntryInfoReturnType>;
+	}): GetContentEntryInfoReturnType | Promise<GetContentEntryInfoReturnType>;
 	getRenderModule?(
 		this: rollup.PluginContext,
 		params: {
@@ -1183,7 +1193,7 @@ export interface ContentEntryType {
 	contentModuleTypes?: string;
 }
 
-type GetEntryInfoReturnType = {
+type GetContentEntryInfoReturnType = {
 	data: Record<string, unknown>;
 	/**
 	 * Used for error hints to point to correct line and location
@@ -1195,12 +1205,23 @@ type GetEntryInfoReturnType = {
 	slug: string;
 };
 
+export interface DataEntryType {
+	extensions: string[];
+	getEntryInfo(params: {
+		fileUrl: URL;
+		contents: string;
+	}): GetDataEntryInfoReturnType | Promise<GetDataEntryInfoReturnType>;
+}
+
+export type GetDataEntryInfoReturnType = { data: Record<string, unknown>; rawData?: string };
+
 export interface AstroSettings {
 	config: AstroConfig;
 	adapter: AstroAdapter | undefined;
 	injectedRoutes: InjectedRoute[];
 	pageExtensions: string[];
 	contentEntryTypes: ContentEntryType[];
+	dataEntryTypes: DataEntryType[];
 	renderers: AstroRenderer[];
 	scripts: {
 		stage: InjectedScriptStage;

packages/astro/src/content/consts.ts

@@ -1,5 +1,8 @@
 export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets';
-export const CONTENT_FLAG = 'astroContent';
+export const CONTENT_FLAG = 'astroContentCollectionEntry';
+export const DATA_FLAG = 'astroDataCollectionEntry';
+export const CONTENT_FLAGS = [CONTENT_FLAG, DATA_FLAG, PROPAGATED_ASSET_FLAG] as const;
+
 export const VIRTUAL_MODULE_ID = 'astro:content';
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
 export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@';

packages/astro/src/content/runtime-assets.ts

@@ -4,7 +4,7 @@ import type { AstroSettings } from '../@types/astro.js';
 import { emitESMImage } from '../assets/index.js';
 
 export function createImage(
-	settings: AstroSettings,
+	settings: Pick<AstroSettings, 'config'>,
 	pluginContext: PluginContext,
 	entryFilePath: string
 ) {

packages/astro/src/content/runtime.ts

@@ -1,6 +1,6 @@
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
 import { prependForwardSlash } from '../core/path.js';
-
+import { ZodIssueCode, string as zodString, type z } from 'zod';
 import {
 	createComponent,
 	createHeadAndContent,
@@ -10,6 +10,7 @@ import {
 	renderUniqueStylesheet,
 	unescapeHTML,
 } from '../runtime/server/index.js';
+import type { ContentLookupMap } from './utils.js';
 
 type LazyImport = () => Promise<any>;
 type GlobResult = Record<string, LazyImport>;
@@ -37,14 +38,31 @@ export function createCollectionToGlobResultMap({
 
 const cacheEntriesByCollection = new Map<string, any[]>();
 export function createGetCollection({
-	collectionToEntryMap,
+	contentCollectionToEntryMap,
+	dataCollectionToEntryMap,
 	getRenderEntryImport,
 }: {
-	collectionToEntryMap: CollectionToEntryMap;
+	contentCollectionToEntryMap: CollectionToEntryMap;
+	dataCollectionToEntryMap: CollectionToEntryMap;
 	getRenderEntryImport: GetEntryImport;
 }) {
 	return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
-		const lazyImports = Object.values(collectionToEntryMap[collection] ?? {});
+		let type: 'content' | 'data';
+		if (collection in contentCollectionToEntryMap) {
+			type = 'content';
+		} else if (collection in dataCollectionToEntryMap) {
+			type = 'data';
+		} else {
+			throw new AstroError({
+				...AstroErrorData.CollectionDoesNotExistError,
+				message: AstroErrorData.CollectionDoesNotExistError.message(collection),
+			});
+		}
+		const lazyImports = Object.values(
+			type === 'content'
+				? contentCollectionToEntryMap[collection]
+				: dataCollectionToEntryMap[collection]
+		);
 		let entries: any[] = [];
 		// Cache `getCollection()` calls in production only
 		// prevents stale cache in development
@@ -54,20 +72,26 @@ export function createGetCollection({
 			entries = await Promise.all(
 				lazyImports.map(async (lazyImport) => {
 					const entry = await lazyImport();
-					return {
-						id: entry.id,
-						slug: entry.slug,
-						body: entry.body,
-						collection: entry.collection,
-						data: entry.data,
-						async render() {
-							return render({
+					return type === 'content'
+						? {
+								id: entry.id,
+								slug: entry.slug,
+								body: entry.body,
 								collection: entry.collection,
+								data: entry.data,
+								async render() {
+									return render({
+										collection: entry.collection,
+										id: entry.id,
+										renderEntryImport: await getRenderEntryImport(collection, entry.slug),
+									});
+								},
+						  }
+						: {
 								id: entry.id,
-								renderEntryImport: await getRenderEntryImport(collection, entry.slug),
-							});
-						},
-					};
+								collection: entry.collection,
+								data: entry.data,
+						  };
 				})
 			);
 			cacheEntriesByCollection.set(collection, entries);
@@ -110,6 +134,92 @@ export function createGetEntryBySlug({
 	};
 }
 
+export function createGetDataEntryById({
+	dataCollectionToEntryMap,
+}: {
+	dataCollectionToEntryMap: CollectionToEntryMap;
+}) {
+	return async function getDataEntryById(collection: string, id: string) {
+		const lazyImport =
+			dataCollectionToEntryMap[collection]?.[/*TODO: filePathToIdMap*/ id + '.json'];
+
+		// TODO: AstroError
+		if (!lazyImport) throw new Error(`Entry ${collection} → ${id} was not found.`);
+		const entry = await lazyImport();
+
+		return {
+			id: entry.id,
+			collection: entry.collection,
+			data: entry.data,
+		};
+	};
+}
+
+export function createGetEntry({
+	getEntryImport,
+	getRenderEntryImport,
+}: {
+	getEntryImport: GetEntryImport;
+	getRenderEntryImport: GetEntryImport;
+}) {
+	return async function getEntry(
+		collectionOrLookupObject:
+			| string
+			| { collection: string; id: string }
+			| { collection: string; slug: string },
+		lookupId?: string
+	) {
+		let collection: string, id: string;
+		if (typeof collectionOrLookupObject === 'string') {
+			collection = collectionOrLookupObject;
+			id = lookupId!;
+		} else {
+			collection = collectionOrLookupObject.collection;
+			id =
+				'id' in collectionOrLookupObject
+					? collectionOrLookupObject.id
+					: collectionOrLookupObject.slug;
+		}
+
+		const entryImport = await getEntryImport(collection, id);
+		if (typeof entryImport !== 'function') return undefined;
+
+		const entry = await entryImport();
+
+		if (entry._internal.type === 'content') {
+			return {
+				id: entry.id,
+				slug: entry.slug,
+				body: entry.body,
+				collection: entry.collection,
+				data: entry.data,
+				async render() {
+					return render({
+						collection: entry.collection,
+						id: entry.id,
+						renderEntryImport: await getRenderEntryImport(collection, id),
+					});
+				},
+			};
+		} else if (entry._internal.type === 'data') {
+			return {
+				id: entry.id,
+				collection: entry.collection,
+				data: entry.data,
+			};
+		}
+		return undefined;
+	};
+}
+
+export function createGetEntries(getEntry: ReturnType<typeof createGetEntry>) {
+	return async function getEntries(
+		entries: { collection: string; id: string }[] | { collection: string; slug: string }[]
+	) {
+		return Promise.all(entries.map((e) => getEntry(e)));
+	};
+}
+
 async function render({
 	collection,
 	id,
@@ -186,3 +296,38 @@ async function render({
 		remarkPluginFrontmatter: mod.frontmatter ?? {},
 	};
 }
+
+export function createReference({ lookupMap }: { lookupMap: ContentLookupMap }) {
+	return function reference(collection: string) {
+		return zodString().transform((lookupId: string, ctx) => {
+			const flattenedErrorPath = ctx.path.join('.');
+			if (!lookupMap[collection]) {
+				ctx.addIssue({
+					code: ZodIssueCode.custom,
+					message: `**${flattenedErrorPath}:** Reference to ${collection} invalid. Collection does not exist or is empty.`,
+				});
+				return;
+			}
+
+			const { type, entries } = lookupMap[collection];
+			const entry = entries[lookupId];
+
+			if (!entry) {
+				ctx.addIssue({
+					code: ZodIssueCode.custom,
+					message: `**${flattenedErrorPath}**: Reference to ${collection} invalid. Expected ${Object.keys(
+						entries
+					)
+						.map((c) => JSON.stringify(c))
+						.join(' | ')}. Received ${JSON.stringify(lookupId)}.`,
+				});
+				return;
+			}
+			// Content is still identified by slugs, so map to a `slug` key for consistency.
+			if (type === 'content') {
+				return { slug: lookupId, collection };
+			}
+			return { id: lookupId, collection };
+		});
+	};
+}