diff --git a/examples/with-data/src/content/authors/ben.json b/examples/with-data/src/data/authors/ben.json similarity index 100% rename from examples/with-data/src/content/authors/ben.json rename to examples/with-data/src/data/authors/ben.json diff --git a/examples/with-data/src/content/banners/welcome.json b/examples/with-data/src/data/banners/welcome.json similarity index 100% rename from examples/with-data/src/content/banners/welcome.json rename to examples/with-data/src/data/banners/welcome.json diff --git a/packages/astro/src/content/index.ts b/packages/astro/src/content/index.ts index 92c8cbdec617..8804b69efca8 100644 --- a/packages/astro/src/content/index.ts +++ b/packages/astro/src/content/index.ts @@ -1,7 +1,7 @@ export { CONTENT_FLAG, PROPAGATED_ASSET_FLAG } from './consts.js'; export { errorMap } from './error-map.js'; export { attachContentServerListeners } from './server-listeners.js'; -export { createContentTypesGenerator } from './types-generator.js'; +export { createCollectionTypesGenerator } from './types-generator.js'; export { contentObservable, getContentPaths, getDotAstroTypeReference } from './utils.js'; export { astroContentAssetPropagationPlugin } from './vite-plugin-content-assets.js'; export { astroContentImportPlugin } from './vite-plugin-content-imports.js'; diff --git a/packages/astro/src/content/runtime.ts b/packages/astro/src/content/runtime.ts index 9ca2048d830d..b93af3b27407 100644 --- a/packages/astro/src/content/runtime.ts +++ b/packages/astro/src/content/runtime.ts @@ -20,15 +20,15 @@ type GetEntryImport = (collection: string, lookupId: string) => Promise + fs.existsSync(p) + ); - if (fs.existsSync(contentPaths.contentDir)) { + if (existentDirs.length) { info( logging, 'content', - `Watching ${cyan( - contentPaths.contentDir.href.replace(settings.config.root.href, '') + `Watching ${bold( + cyan(existentDirs.map((dir) => rootRelativePath(root, dir, false)).join(' and ')) )} for changes` ); const maybeTsConfigStats = getTSConfigStatsWhenAllowJsFalse({ contentPaths, settings }); if (maybeTsConfigStats) warnAllowJsIsFalse({ ...maybeTsConfigStats, logging }); await attachListeners(); } else { - viteServer.watcher.on('addDir', contentDirListener); - async function contentDirListener(dir: string) { - if (appendForwardSlash(pathToFileURL(dir).href) === contentPaths.contentDir.href) { - info(logging, 'content', `Content dir found. Watching for changes`); - await attachListeners(); - viteServer.watcher.removeListener('addDir', contentDirListener); + viteServer.watcher.on('addDir', dirListeners); + // TODO: clean up a bit + let attached = false; + let foundContentDir = false; + let foundDataDir = false; + async function dirListeners(dir: string) { + const collectionDir = getCollectionDirByUrl(pathToFileURL(dir), contentPaths); + if (collectionDir === 'content') foundContentDir = true; + if (collectionDir === 'data') foundDataDir = true; + if (collectionDir) { + info( + logging, + 'content', + `Watching ${cyan( + rootRelativePath( + collectionDir === 'content' ? contentPaths.contentDir : contentPaths.dataDir, + root + ) + )} for changes` + ); + if (!attached) { + await attachListeners(); + attached = true; + } + if (foundContentDir && foundDataDir) { + viteServer.watcher.removeListener('addDir', dirListeners); + } } } } async function attachListeners() { - const contentGenerator = await createContentTypesGenerator({ + const typesGenerator = await createCollectionTypesGenerator({ fs, settings, logging, viteServer, contentConfigObserver: globalContentConfigObserver, }); - await contentGenerator.init(); + await typesGenerator.init(); info(logging, 'content', 'Types generated'); viteServer.watcher.on('add', (entry) => { - contentGenerator.queueEvent({ name: 'add', entry }); + typesGenerator.queueEvent({ name: 'add', entry }); }); viteServer.watcher.on('addDir', (entry) => - contentGenerator.queueEvent({ name: 'addDir', entry }) + typesGenerator.queueEvent({ name: 'addDir', entry }) ); viteServer.watcher.on('change', (entry) => - contentGenerator.queueEvent({ name: 'change', entry }) + typesGenerator.queueEvent({ name: 'change', entry }) ); viteServer.watcher.on('unlink', (entry) => { - contentGenerator.queueEvent({ name: 'unlink', entry }); + typesGenerator.queueEvent({ name: 'unlink', entry }); }); viteServer.watcher.on('unlinkDir', (entry) => - contentGenerator.queueEvent({ name: 'unlinkDir', entry }) + typesGenerator.queueEvent({ name: 'unlinkDir', entry }) ); } } diff --git a/packages/astro/src/content/template/virtual-mod.mjs b/packages/astro/src/content/template/virtual-mod.mjs index 113b46d7d0f3..e3df545437c0 100644 --- a/packages/astro/src/content/template/virtual-mod.mjs +++ b/packages/astro/src/content/template/virtual-mod.mjs @@ -26,13 +26,14 @@ export const image = () => { }; const contentDir = '@@CONTENT_DIR@@'; +const dataDir = '@@DATA_DIR@@'; const contentEntryGlob = import.meta.glob('@@CONTENT_ENTRY_GLOB_PATH@@', { query: { astroContentCollectionEntry: true }, }); const contentCollectionToEntryMap = createCollectionToGlobResultMap({ globResult: contentEntryGlob, - contentDir, + dir: contentDir, }); const dataEntryGlob = import.meta.glob('@@DATA_ENTRY_GLOB_PATH@@', { @@ -40,7 +41,7 @@ const dataEntryGlob = import.meta.glob('@@DATA_ENTRY_GLOB_PATH@@', { }); const dataCollectionToEntryMap = createCollectionToGlobResultMap({ globResult: dataEntryGlob, - contentDir, + dir: dataDir, }); let lookupMap = {}; @@ -60,7 +61,7 @@ const renderEntryGlob = import.meta.glob('@@RENDER_ENTRY_GLOB_PATH@@', { }); const collectionToRenderEntryMap = createCollectionToGlobResultMap({ globResult: renderEntryGlob, - contentDir, + dir: contentDir, }); export const getCollection = createGetCollection({ diff --git a/packages/astro/src/content/types-generator.ts b/packages/astro/src/content/types-generator.ts index 7a5407ae915a..f20a30c1deb9 100644 --- a/packages/astro/src/content/types-generator.ts +++ b/packages/astro/src/content/types-generator.ts @@ -22,11 +22,13 @@ import { getEntryCollectionName, getDataEntryExts, getDataEntryId, + getCollectionDirByUrl, } from './utils.js'; +import { rootRelativePath } from '../core/util.js'; type ChokidarEvent = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir'; type RawContentEvent = { name: ChokidarEvent; entry: string }; -type ContentEvent = { name: ChokidarEvent; entry: URL }; +type ContentEvent = { name: ChokidarEvent; entry: URL; collectionDir: 'content' | 'data' }; type ContentEntryMetadata = { type: 'content'; slug: string }; type DataEntryMetadata = { type: 'data' }; @@ -45,13 +47,13 @@ type CreateContentGeneratorParams = { type EventOpts = { logLevel: 'info' | 'warn' }; type EventWithOptions = { - type: ContentEvent; + event: ContentEvent; opts: EventOpts | undefined; }; class UnsupportedFileTypeError extends Error {} -export async function createContentTypesGenerator({ +export async function createCollectionTypesGenerator({ contentConfigObserver, fs, logging, @@ -70,32 +72,39 @@ export async function createContentTypesGenerator({ const typeTemplateContent = await fs.promises.readFile(contentPaths.typesTemplate, 'utf-8'); async function init(): Promise< - { typesGenerated: true } | { typesGenerated: false; reason: 'no-content-dir' } + { typesGenerated: true } | { typesGenerated: false; reason: 'no-dirs' } > { - if (!fs.existsSync(contentPaths.contentDir)) { - return { typesGenerated: false, reason: 'no-content-dir' }; + if (!fs.existsSync(contentPaths.contentDir) && !fs.existsSync(contentPaths.dataDir)) { + return { typesGenerated: false, reason: 'no-dirs' }; } events.push({ - type: { name: 'add', entry: contentPaths.config.url }, + event: { name: 'add', entry: contentPaths.config.url, collectionDir: 'content' }, opts: { logLevel: 'warn' }, }); - const globResult = await glob('**', { - cwd: fileURLToPath(contentPaths.contentDir), + const relContentDir = rootRelativePath(settings.config.root, contentPaths.contentDir, false); + const relDataDir = rootRelativePath(settings.config.root, contentPaths.dataDir, false); + + const globResult = await glob([relContentDir + '**', relDataDir + '**'], { + absolute: true, + cwd: fileURLToPath(settings.config.root), fs: { readdir: fs.readdir.bind(fs), readdirSync: fs.readdirSync.bind(fs), }, }); const entries = globResult - .map((e) => new URL(e, contentPaths.contentDir)) + .map((e) => pathToFileURL(e)) .filter( // Config loading handled first. Avoid running twice. (e) => !e.href.startsWith(contentPaths.config.url.href) ); for (const entry of entries) { - events.push({ type: { name: 'add', entry }, opts: { logLevel: 'warn' } }); + events.push({ + event: { name: 'add', entry, collectionDir: getCollectionDirByUrl(entry, contentPaths)! }, + opts: { logLevel: 'warn' }, + }); } await runEvents(); return { typesGenerated: true }; @@ -132,6 +141,7 @@ export async function createContentTypesGenerator({ contentPaths, contentEntryExts, dataEntryExts, + event.collectionDir, settings.config.experimental.assets ); if (fileType === 'ignored') { @@ -173,18 +183,20 @@ export async function createContentTypesGenerator({ } const { entry } = event; - const { contentDir } = contentPaths; + const collectionDir = + event.collectionDir === 'content' ? contentPaths.contentDir : contentPaths.dataDir; - const collection = getEntryCollectionName({ entry, contentDir }); + const collection = getEntryCollectionName({ + entry, + dir: collectionDir, + }); if (collection === undefined) { if (['info', 'warn'].includes(logLevel)) { warn( logging, 'content', `${cyan( - normalizePath( - path.relative(fileURLToPath(contentPaths.contentDir), fileURLToPath(event.entry)) - ) + normalizePath(path.relative(fileURLToPath(collectionDir), fileURLToPath(event.entry))) )} must be nested in a collection directory. Skipping.` ); } @@ -192,7 +204,7 @@ export async function createContentTypesGenerator({ } if (fileType === 'data') { - const id = getDataEntryId({ entry, contentDir, collection }); + const id = getDataEntryId({ entry, dataDir: contentPaths.dataDir, collection }); const collectionKey = JSON.stringify(collection); const entryKey = JSON.stringify(id); @@ -217,7 +229,11 @@ export async function createContentTypesGenerator({ const contentEntryType = contentEntryConfigByExt.get(path.extname(event.entry.pathname)); if (!contentEntryType) return { shouldGenerateTypes: false }; - const { id, slug: generatedSlug } = getContentEntryIdAndSlug({ entry, contentDir, collection }); + const { id, slug: generatedSlug } = getContentEntryIdAndSlug({ + entry, + contentDir: contentPaths.contentDir, + collection, + }); const collectionKey = JSON.stringify(collection); const entryKey = JSON.stringify(id); @@ -268,16 +284,14 @@ export async function createContentTypesGenerator({ } function queueEvent(rawEvent: RawContentEvent, opts?: EventOpts) { - const event = { - type: { - entry: pathToFileURL(rawEvent.entry), - name: rawEvent.name, - }, - opts, - }; - if (!event.type.entry.pathname.startsWith(contentPaths.contentDir.pathname)) return; + const entryUrl = pathToFileURL(rawEvent.entry); + const collectionDir = getCollectionDirByUrl(entryUrl, contentPaths); + if (!collectionDir) return; - events.push(event); + events.push({ + event: { name: rawEvent.name, entry: entryUrl, collectionDir: collectionDir }, + opts, + }); debounceTimeout && clearTimeout(debounceTimeout); const runEventsSafe = async () => { @@ -297,7 +311,7 @@ export async function createContentTypesGenerator({ const eventResponses = []; for (const event of events) { - const response = await handleEvent(event.type, event.opts); + const response = await handleEvent(event.event, event.opts); eventResponses.push(response); } diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 8f9c53f43ec2..f64ed7df60c2 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -169,12 +169,9 @@ export function getContentEntryConfigByExtMap(settings: Pick & { entry: string | URL }) { +export function getEntryCollectionName({ dir, entry }: { dir: URL; entry: string | URL }) { const entryPath = typeof entry === 'string' ? entry : fileURLToPath(entry); - const rawRelativePath = path.relative(fileURLToPath(contentDir), entryPath); + const rawRelativePath = path.relative(fileURLToPath(dir), entryPath); const collectionName = path.dirname(rawRelativePath).split(path.sep).shift() ?? ''; const isOutsideCollection = collectionName === '' || collectionName === '..' || collectionName === '.'; @@ -188,10 +185,14 @@ export function getEntryCollectionName({ export function getDataEntryId({ entry, - contentDir, + dataDir, collection, -}: Pick & { entry: URL; collection: string }): string { - const rawRelativePath = path.relative(fileURLToPath(contentDir), fileURLToPath(entry)); +}: { + dataDir: URL; + entry: URL; + collection: string; +}): string { + const rawRelativePath = path.relative(fileURLToPath(dataDir), fileURLToPath(entry)); const rawId = path.relative(collection, rawRelativePath); const rawIdWithoutFileExt = rawId.replace(new RegExp(path.extname(rawId) + '$'), ''); @@ -202,7 +203,11 @@ export function getContentEntryIdAndSlug({ entry, contentDir, collection, -}: Pick & { entry: URL; collection: string }): { +}: { + contentDir: URL; + entry: URL; + collection: string; +}): { id: string; slug: string; } { @@ -227,24 +232,29 @@ export function getContentEntryIdAndSlug({ export function getEntryType( entryPath: string, - paths: Pick, + paths: Pick, contentFileExts: string[], dataFileExts: string[], + collectionDir?: 'content' | 'data', // TODO: Unflag this when we're ready to release assets - erika, 2023-04-12 - experimentalAssets: boolean + experimentalAssets: boolean = false ): 'content' | 'data' | 'config' | 'ignored' | 'unsupported' { const { ext, base } = path.parse(entryPath); const fileUrl = pathToFileURL(entryPath); if ( - hasUnderscoreBelowContentDirectoryPath(fileUrl, paths.contentDir) || - isOnIgnoreList(base) || - (experimentalAssets && isImageAsset(ext)) + (experimentalAssets && isImageAsset(ext)) || + (collectionDir && + (hasUnderscoreBelowDirectoryPath( + fileUrl, + collectionDir === 'content' ? paths.contentDir : paths.dataDir + ) || + isOnIgnoreList(base))) ) { return 'ignored'; - } else if (contentFileExts.includes(ext)) { + } else if (contentFileExts.includes(ext) && collectionDir === 'content') { return 'content'; - } else if (dataFileExts.includes(ext)) { + } else if (dataFileExts.includes(ext) && collectionDir === 'data') { return 'data'; } else if (fileUrl.href === paths.config.url.href) { return 'config'; @@ -264,11 +274,8 @@ function isImageAsset(fileExt: string) { return VALID_INPUT_FORMATS.includes(fileExt.slice(1) as ImageInputFormat); } -function hasUnderscoreBelowContentDirectoryPath( - fileUrl: URL, - contentDir: ContentPaths['contentDir'] -): boolean { - const parts = fileUrl.pathname.replace(contentDir.pathname, '').split('/'); +function hasUnderscoreBelowDirectoryPath(fileUrl: URL, dir: URL): boolean { + const parts = fileUrl.pathname.replace(dir.pathname, '').split('/'); for (const part of parts) { if (part.startsWith('_')) return true; } @@ -319,7 +326,7 @@ export const globalContentConfigObserver = contentObservable({ status: 'init' }) const InvalidDataCollectionConfigError = { ...AstroErrorData.UnknownContentCollectionError, message: (dataExtsStringified: string, collection: string) => - `Found a non-data collection with ${dataExtsStringified} files: **${collection}.** To make this a data collection, use the \`defineDataCollection()\` helper or add \`type: 'data'\` to your collection config.`, + `Found a non-data collection with ${dataExtsStringified} files: **${collection}.** To make this a data collection, 1) move to \`src/data/\`, and 2) use the \`defineDataCollection()\` helper or add \`type: 'data'\` to your collection config.`, }; export function hasContentFlag(viteId: string, flag: (typeof CONTENT_FLAGS)[number]) { @@ -352,7 +359,7 @@ export async function loadContentConfig({ // Check that data collections are properly configured using `defineDataCollection()`. const dataEntryExts = getDataEntryExts(settings); - const collectionEntryGlob = await glob('**', { + const contentCollectionGlob = await glob('**', { cwd: fileURLToPath(contentPaths.contentDir), absolute: true, fs: { @@ -361,8 +368,8 @@ export async function loadContentConfig({ }, }); - for (const entry of collectionEntryGlob) { - const collectionName = getEntryCollectionName({ contentDir: contentPaths.contentDir, entry }); + for (const entry of contentCollectionGlob) { + const collectionName = getEntryCollectionName({ dir: contentPaths.contentDir, entry }); if (!collectionName || !config.data.collections[collectionName]) continue; const { type } = config.data.collections[collectionName]; @@ -383,6 +390,18 @@ export async function loadContentConfig({ } } +export function getCollectionDirByUrl( + url: URL, + contentPaths: Pick +): 'content' | 'data' | undefined { + if (url.href.startsWith(contentPaths.contentDir.href)) { + return 'content'; + } else if (url.href.startsWith(contentPaths.dataDir.href)) { + return 'data'; + } + return undefined; +} + type ContentCtx = | { status: 'init' } | { status: 'loading' } @@ -424,6 +443,7 @@ export function contentObservable(initialCtx: ContentCtx): ContentObservable { export type ContentPaths = { contentDir: URL; + dataDir: URL; assetsDir: URL; cacheDir: URL; typesTemplate: URL; @@ -444,6 +464,7 @@ export function getContentPaths( return { cacheDir: new URL('.astro/', root), contentDir: new URL('./content/', srcDir), + dataDir: new URL('./data/', srcDir), assetsDir: new URL('./assets/', srcDir), typesTemplate: new URL('types.d.ts', templateDir), virtualModTemplate: new URL('virtual-mod.mjs', templateDir), @@ -497,7 +518,10 @@ export async function getStringifiedLookupMap({ contentGlob.map(async (filePath) => { const contentEntryType = contentEntryConfigByExt.get(extname(filePath)); if (!contentEntryType) return; - const collection = getEntryCollectionName({ contentDir, entry: pathToFileURL(filePath) }); + const collection = getEntryCollectionName({ + dir: contentDir, + entry: pathToFileURL(filePath), + }); if (!collection) return; const { id, slug: generatedSlug } = await getContentEntryIdAndSlug({ diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts index 57927305ecd8..6e39949b2c12 100644 --- a/packages/astro/src/content/vite-plugin-content-imports.ts +++ b/packages/astro/src/content/vite-plugin-content-imports.ts @@ -255,7 +255,7 @@ export const _internal = { }); const entry = pathToFileURL(fileId); const { contentDir } = contentPaths; - const collection = getEntryCollectionName({ entry, contentDir }); + const collection = getEntryCollectionName({ entry, dir: contentDir }); if (collection === undefined) throw new AstroError(AstroErrorData.UnknownContentCollectionError); @@ -351,7 +351,7 @@ async function getDataEntryModule({ }: { fs: typeof fsMod; fileId: string; - contentPaths: Pick; + contentPaths: Pick; pluginContext: PluginContext; dataEntryExtToParser: Map; settings: Pick; @@ -374,11 +374,11 @@ async function getDataEntryModule({ contents: rawContents, }); const entry = pathToFileURL(fileId); - const { contentDir } = contentPaths; - const collection = getEntryCollectionName({ entry, contentDir }); + const { dataDir } = contentPaths; + const collection = getEntryCollectionName({ entry, dir: dataDir }); if (collection === undefined) throw new AstroError(AstroErrorData.UnknownContentCollectionError); - const id = getDataEntryId({ entry, contentDir, collection }); + const id = getDataEntryId({ entry, dataDir, collection }); const _internal = { filePath: fileId, rawData: '' }; diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index 1007dc788eb1..5a7214eab5ff 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -20,6 +20,7 @@ export function astroContentVirtualModPlugin({ }: AstroContentVirtualModPluginParams): Plugin { const contentPaths = getContentPaths(settings.config); const relContentDir = rootRelativePath(settings.config.root, contentPaths.contentDir); + const relDataDir = rootRelativePath(settings.config.root, contentPaths.dataDir); const contentEntryConfigByExt = getContentEntryConfigByExtMap(settings); const contentEntryExts = [...contentEntryConfigByExt.keys()]; @@ -28,11 +29,14 @@ export function astroContentVirtualModPlugin({ const virtualModContents = fsMod .readFileSync(contentPaths.virtualModTemplate, 'utf-8') .replace('@@CONTENT_DIR@@', relContentDir) + .replace('@@DATA_DIR@@', relDataDir) .replace('@@CONTENT_ENTRY_GLOB_PATH@@', `${relContentDir}**/*${getExtGlob(contentEntryExts)}`) - .replace('@@DATA_ENTRY_GLOB_PATH@@', `${relContentDir}**/*${getExtGlob(dataEntryExts)}`) + .replace('@@DATA_ENTRY_GLOB_PATH@@', `${relDataDir}**/*${getExtGlob(dataEntryExts)}`) .replace( '@@RENDER_ENTRY_GLOB_PATH@@', - `${relContentDir}**/*${getExtGlob(/** Note: data collections excluded */ contentEntryExts)}` + `${relContentDir}**/*${getExtGlob( + contentEntryExts /** data collections excluded since they don't have a `render()` function */ + )}` ); const astroContentVirtualModuleId = '\0' + VIRTUAL_MODULE_ID; diff --git a/packages/astro/src/core/sync/index.ts b/packages/astro/src/core/sync/index.ts index 0017225d0576..728da2de6f10 100644 --- a/packages/astro/src/core/sync/index.ts +++ b/packages/astro/src/core/sync/index.ts @@ -4,7 +4,7 @@ import { performance } from 'node:perf_hooks'; import { createServer } from 'vite'; import type { Arguments } from 'yargs-parser'; import type { AstroSettings } from '../../@types/astro'; -import { createContentTypesGenerator } from '../../content/index.js'; +import { createCollectionTypesGenerator } from '../../content/index.js'; import { globalContentConfigObserver } from '../../content/utils.js'; import { runHookConfigSetup } from '../../integrations/index.js'; import { setUpEnvTs } from '../../vite-plugin-inject-env-ts/index.js'; @@ -75,7 +75,7 @@ export async function sync( ); try { - const contentTypesGenerator = await createContentTypesGenerator({ + const contentTypesGenerator = await createCollectionTypesGenerator({ contentConfigObserver: globalContentConfigObserver, logging, fs, @@ -91,7 +91,7 @@ export async function sync( if (typesResult.typesGenerated === false) { switch (typesResult.reason) { - case 'no-content-dir': + case 'no-dirs': default: info(logging, 'content', 'No content directory found. Skipping type generation.'); return 0;