feat: add O(1) lookups for getDataEntryById

packages/astro/src/content/runtime.ts

@@ -15,6 +15,7 @@ import {
 
 type GlobResult = Record<string, () => Promise<any>>;
 type CollectionToEntryMap = Record<string, GlobResult>;
+type GetEntryImport = (collection: string, lookupId: string) => () => Promise<any>;
 
 export function createCollectionToGlobResultMap({
 	globResult,
@@ -29,9 +30,8 @@ export function createCollectionToGlobResultMap({
 		const segments = keyRelativeToDir.split('/');
 		if (segments.length <= 1) continue;
 		const collection = segments[0];
-		const entryId = segments.slice(1).join('/');
 		collectionToGlobResultMap[collection] ??= {};
-		collectionToGlobResultMap[collection][entryId] = globResult[key];
+		collectionToGlobResultMap[collection][key] = globResult[key];
 	}
 	return collectionToGlobResultMap;
 }
@@ -40,11 +40,11 @@ const cacheEntriesByCollection = new Map<string, any[]>();
 export function createGetCollection({
 	contentCollectionToEntryMap,
 	dataCollectionToEntryMap,
-	collectionToRenderEntryMap,
+	getRenderEntryImport,
 }: {
 	contentCollectionToEntryMap: CollectionToEntryMap;
 	dataCollectionToEntryMap: CollectionToEntryMap;
-	collectionToRenderEntryMap: CollectionToEntryMap;
+	getRenderEntryImport: GetEntryImport;
 }) {
 	return async function getCollection(collection: string, filter?: (entry: any) => unknown) {
 		let type: 'content' | 'data';
@@ -83,7 +83,7 @@ export function createGetCollection({
 									return render({
 										collection: entry.collection,
 										id: entry.id,
-										collectionToRenderEntryMap,
+										renderEntryImport: await getRenderEntryImport(collection, entry.slug),
 									});
 								},
 						  }
@@ -106,10 +106,10 @@ export function createGetCollection({
 
 export function createGetEntryBySlug({
 	getCollection,
-	collectionToRenderEntryMap,
+	getRenderEntryImport,
 }: {
 	getCollection: ReturnType<typeof createGetCollection>;
-	collectionToRenderEntryMap: CollectionToEntryMap;
+	getRenderEntryImport: GetEntryImport;
 }) {
 	return async function getEntryBySlug(collection: string, slug: string) {
 		// This is not an optimized lookup. Should look into an O(1) implementation
@@ -138,24 +138,18 @@ export function createGetEntryBySlug({
 				return render({
 					collection: entry.collection,
 					id: entry.id,
-					collectionToRenderEntryMap,
+					renderEntryImport: await getRenderEntryImport(collection, entry.slug),
 				});
 			},
 		};
 	};
 }
 
-export function createGetDataEntryById({
-	dataCollectionToEntryMap,
-}: {
-	dataCollectionToEntryMap: CollectionToEntryMap;
-}) {
+export function createGetDataEntryById({ getEntryImport }: { getEntryImport: GetEntryImport }) {
 	return async function getDataEntryById(collection: string, id: string) {
-		const lazyImport =
-			dataCollectionToEntryMap[collection]?.[/*TODO: filePathToIdMap*/ id + '.json'];
+		const lazyImport = await getEntryImport(collection, id);
 
-		// TODO: AstroError
-		if (!lazyImport) throw new Error(`Entry ${collection} → ${id} was not found.`);
+		if (!lazyImport) return undefined;
 		const entry = await lazyImport();
 
 		return {
@@ -169,21 +163,20 @@ export function createGetDataEntryById({
 async function render({
 	collection,
 	id,
-	collectionToRenderEntryMap,
+	renderEntryImport,
 }: {
 	collection: string;
 	id: string;
-	collectionToRenderEntryMap: CollectionToEntryMap;
+	renderEntryImport?: () => Promise<any>;
 }) {
 	const UnexpectedRenderError = new AstroError({
 		...AstroErrorData.UnknownContentCollectionError,
 		message: `Unexpected error while rendering ${String(collection)} → ${String(id)}.`,
 	});
 
-	const lazyImport = collectionToRenderEntryMap[collection]?.[id];
-	if (typeof lazyImport !== 'function') throw UnexpectedRenderError;
+	if (typeof renderEntryImport !== 'function') throw UnexpectedRenderError;
 
-	const baseMod = await lazyImport();
+	const baseMod = await renderEntryImport();
 	if (baseMod == null || typeof baseMod !== 'object') throw UnexpectedRenderError;
 
 	const { collectedStyles, collectedLinks, collectedScripts, getMod } = baseMod;

packages/astro/src/content/template/virtual-mod.mjs

@@ -28,6 +28,16 @@ const dataCollectionToEntryMap = createCollectionToGlobResultMap({
 	dir: dataDir,
 });
 
+function createGlobLookup(entryGlob) {
+	return async (collection, lookupId) => {
+		const { default: lookupMap } = await import('@@LOOKUP_MAP_PATH@@');
+		const filePath = lookupMap[collection]?.[lookupId];
+
+		if (!filePath) return undefined;
+		return entryGlob[collection][filePath];
+	};
+}
+
 const renderEntryGlob = import.meta.glob('@@RENDER_ENTRY_GLOB_PATH@@', {
 	query: { astroPropagatedAssets: true },
 });
@@ -74,16 +84,16 @@ export const image = () => {
 export const getCollection = createGetCollection({
 	contentCollectionToEntryMap,
 	dataCollectionToEntryMap,
-	collectionToRenderEntryMap,
+	getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
 });
 
 export const getEntryBySlug = createGetEntryBySlug({
-	collectionToRenderEntryMap,
 	getCollection,
+	getRenderEntryImport: createGlobLookup(collectionToRenderEntryMap),
 });
 
 export const getDataEntryById = createGetDataEntryById({
-	dataCollectionToEntryMap,
+	getEntryImport: createGlobLookup(dataCollectionToEntryMap),
 });
 
 export const reference = createReference({

packages/astro/src/content/types-generator.ts

@@ -23,6 +23,7 @@ import {
 	getDataEntryId,
 	getCollectionDirByUrl,
 	reloadContentConfigObserver,
+	updateLookupMaps,
 } from './utils.js';
 import { rootRelativePath } from '../core/util.js';
 
@@ -321,6 +322,13 @@ export async function createCollectionTypesGenerator({
 				contentConfig: observable.status === 'loaded' ? observable.config : undefined,
 				contentEntryTypes: settings.contentEntryTypes,
 			});
+			await updateLookupMaps({
+				contentEntryExts,
+				dataEntryExts,
+				contentPaths,
+				root: settings.config.root,
+				fs,
+			});
 			if (observable.status === 'loaded' && ['info', 'warn'].includes(logLevel)) {
 				warnNonexistentCollections({
 					logging,

packages/astro/src/content/utils.ts

@@ -1,4 +1,4 @@
-import glob from 'fast-glob';
+import glob, { type Options as FastGlobOptions } from 'fast-glob';
 import { slug as githubSlug } from 'github-slugger';
 import matter from 'gray-matter';
 import fsMod from 'node:fs';
@@ -18,6 +18,7 @@ import { AstroError, AstroErrorData } from '../core/errors/index.js';
 import { CONTENT_TYPES_FILE, CONTENT_FLAGS } from './consts.js';
 import { errorMap } from './error-map.js';
 import { createImage } from './runtime-assets.js';
+import { rootRelativePath } from '../core/util.js';
 
 export const collectionConfigParser = z.union([
 	z.object({
@@ -188,10 +189,13 @@ export function getDataEntryId({
 	collection,
 }: {
 	dataDir: URL;
-	entry: URL;
+	entry: string | URL;
 	collection: string;
 }): string {
-	const rawRelativePath = path.relative(fileURLToPath(dataDir), fileURLToPath(entry));
+	const rawRelativePath = path.relative(
+		fileURLToPath(dataDir),
+		typeof entry === 'string' ? entry : fileURLToPath(entry)
+	);
 	const rawId = path.relative(collection, rawRelativePath);
 	const rawIdWithoutFileExt = rawId.replace(new RegExp(path.extname(rawId) + '$'), '');
 
@@ -204,13 +208,16 @@ export function getContentEntryIdAndSlug({
 	collection,
 }: {
 	contentDir: URL;
-	entry: URL;
+	entry: string | URL;
 	collection: string;
 }): {
 	id: string;
 	slug: string;
 } {
-	const rawRelativePath = path.relative(fileURLToPath(contentDir), fileURLToPath(entry));
+	const rawRelativePath = path.relative(
+		fileURLToPath(contentDir),
+		typeof entry === 'string' ? entry : fileURLToPath(entry)
+	);
 	const rawId = path.relative(collection, rawRelativePath);
 	const rawIdWithoutFileExt = rawId.replace(new RegExp(path.extname(rawId) + '$'), '');
 	const rawSlugSegments = rawIdWithoutFileExt.split(path.sep);
@@ -372,7 +379,6 @@ export async function loadContentConfig({
 
 		const contentCollectionGlob = await glob('**', {
 			cwd: fileURLToPath(contentPaths.contentDir),
-			absolute: true,
 			fs: {
 				readdir: fs.readdir.bind(fs),
 				readdirSync: fs.readdirSync.bind(fs),
@@ -519,3 +525,65 @@ function search(fs: typeof fsMod, srcDir: URL) {
 	}
 	return { exists: false, url: paths[0] };
 }
+
+export async function updateLookupMaps({
+	contentPaths,
+	contentEntryExts,
+	dataEntryExts,
+	root,
+	fs,
+}: {
+	contentEntryExts: string[];
+	dataEntryExts: string[];
+	contentPaths: ContentPaths;
+	root: URL;
+	fs: typeof fsMod;
+}) {
+	console.log('starting...');
+	const { contentDir, dataDir } = contentPaths;
+	const globOpts: FastGlobOptions = {
+		absolute: false,
+		cwd: fileURLToPath(root),
+		fs: {
+			readdir: fs.readdir.bind(fs),
+			readdirSync: fs.readdirSync.bind(fs),
+		},
+	};
+
+	const relContentDir = rootRelativePath(root, contentDir, false);
+	const contentGlob = await glob(`${relContentDir}/**/*${getExtGlob(contentEntryExts)}`, globOpts);
+	let filePathByLookupId: {
+		[collection: string]: Record<string, string>;
+	} = {};
+
+	for (const filePath of contentGlob) {
+		const collection = getEntryCollectionName({ dir: contentDir, entry: filePath });
+		if (!collection) continue;
+		const { slug } = getContentEntryIdAndSlug({ collection, contentDir, entry: filePath });
+		filePathByLookupId[collection] ??= {};
+		filePathByLookupId[collection][slug] = '/' + filePath;
+	}
+
+	const relDataDir = rootRelativePath(root, dataDir, false);
+	const dataGlob = await glob(`${relDataDir}/**/*${getExtGlob(dataEntryExts)}`, globOpts);
+
+	for (const filePath of dataGlob) {
+		const collection = getEntryCollectionName({ dir: dataDir, entry: filePath });
+		if (!collection) continue;
+		const id = getDataEntryId({ entry: filePath, collection, dataDir });
+		filePathByLookupId[collection] ??= {};
+		filePathByLookupId[collection][id] = '/' + filePath;
+	}
+
+	await fs.promises.writeFile(
+		new URL('lookup-map.json', contentPaths.cacheDir),
+		JSON.stringify(filePathByLookupId, null, 2)
+	);
+}
+
+export function getExtGlob(exts: string[]) {
+	return exts.length === 1
+		? // Wrapping {...} breaks when there is only one extension
+		  exts[0]
+		: `{${exts.join(',')}}`;
+}

packages/astro/src/content/vite-plugin-content-virtual-mod.ts

@@ -1,11 +1,8 @@
 import fsMod from 'node:fs';
-import * as path from 'node:path';
 import type { Plugin } from 'vite';
-import { normalizePath } from 'vite';
 import type { AstroSettings } from '../@types/astro.js';
-import { appendForwardSlash, prependForwardSlash } from '../core/path.js';
 import { VIRTUAL_MODULE_ID } from './consts.js';
-import { getContentEntryExts, getContentPaths, getDataEntryExts } from './utils.js';
+import { getContentEntryExts, getContentPaths, getDataEntryExts, getExtGlob } from './utils.js';
 import { rootRelativePath } from '../core/util.js';
 
 interface AstroContentVirtualModPluginParams {
@@ -27,6 +24,7 @@ export function astroContentVirtualModPlugin({
 			'@@COLLECTION_NAME_BY_REFERENCE_KEY@@',
 			new URL('reference-map.json', contentPaths.cacheDir).pathname
 		)
+		.replace('@@LOOKUP_MAP_PATH@@', new URL('lookup-map.json', contentPaths.cacheDir).pathname)
 		.replace('@@CONTENT_DIR@@', relContentDir)
 		.replace('@@DATA_DIR@@', relDataDir)
 		.replace('@@CONTENT_ENTRY_GLOB_PATH@@', `${relContentDir}**/*${getExtGlob(contentEntryExts)}`)
@@ -57,10 +55,3 @@ export function astroContentVirtualModPlugin({
 		},
 	};
 }
-
-function getExtGlob(exts: string[]) {
-	return exts.length === 1
-		? // Wrapping {...} breaks when there is only one extension
-		  exts[0]
-		: `{${exts.join(',')}}`;
-}