From 9502a4f2c06e6e3990050f06bab97e377e54fa90 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 2 May 2025 17:38:00 +0100 Subject: [PATCH 01/14] Add live content loaders RFC --- proposals/0055-live-content-loaders.md | 383 +++++++++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 proposals/0055-live-content-loaders.md diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md new file mode 100644 index 00000000..3e84b5db --- /dev/null +++ b/proposals/0055-live-content-loaders.md @@ -0,0 +1,383 @@ + + +**If you have feedback and the feature is released as experimental, please leave it on the Stage 3 PR. Otherwise, comment on the Stage 2 issue (links below).** + +- Start Date: 2025-05-02 +- Reference Issues: +- Implementation PR: https://github.com/withastro/astro/pull/13685 +- Stage 1 Discussion: https://github.com/withastro/roadmap/discussions/1137 +- Stage 2 Issue: https://github.com/withastro/roadmap/issues/1151 +- Stage 3 PR: + +# Summary + +Adds support for live data to content collections. Defines a new type of content loader that fetches data at runtime rather than build time, allowing users to get the data with a similar API. + +# Example + +Defining a live loader for a store API: + +```ts +// storeloader.ts +import { type Product, loadStoreData } from "./lib/api.ts"; + +interface StoreCollectionFilter { + category?: string; +} + +interface StoreEntryFilter { + slug?: string; +} + +export function storeLoader({ + field, + key, +}): LiveLoader { + return { + name: "store-loader", + loadCollection: async ({ logger, filter }) => { + logger.info(`Loading collection from ${field}`); + // load from API + const products = await loadStoreData({ field, key, filter }); + const entries = products.map((product) => ({ + id: product.id, + data: product, + })); + return { + entries, + }; + }, + loadEntry: async ({ logger, filter }) => { + logger.info(`Loading entry from ${field}`); + // load from API + const product = await loadStoreData({ + field, + key, + filter, + }); + + if (!product) { + logger.error(`Product not found`); + return; + } + return { + id: filter.id, + data: product, + }; + }, + }; +} +``` + +A new `src/live.config.ts` file is introduced that uses the same syntax as the `src/content.config.ts` file: + +```ts +// src/live.config.ts +import { defineCollection } from "astro:content"; + +import { storeLoader } from "@mystore/astro-loader"; + +const products = defineCollection({ + type: "live", + loader: storeLoader({ field: "products", key: process.env.STORE_KEY }), +}); + +export const collections = { products }; +``` + +The loader can be used in the same way as a normal content collection: + +```astro +--- +import { getCollection, getEntry } from "astro:content"; + +// Get all entries in a collection, like other collections +const allProducts = await getCollection("products"); + +// Live collections optionally allow extra filters to be passed in, defined by the loader +const clothes = await getCollection("products", { category: "clothes" }); + +// Get entrey by ID like other collections +const productById = await getEntry("products", Astro.params.id); + +// Query a single entry using the object syntax +const productBySlug = await getEntry("products", { slug: Astro.params.slug }); +--- +``` + +# Background & Motivation + +In Astro 5, the content layer API added support for adding diverse content sources to content collections. Users can create loaders that fetch data from any source at build time, and then access it inside a page via `getEntry` and `getCollection`. The data is cached between builds, giving fast access and updates. However there is no method for updating the data store between builds, meaning any updates to the data need a full site deploy, even if the pages are rendered on-demand. + +This means that content collections are not suitable for pages that update frequently. Instead, today these pages tend to access the APIs directly in the frontmatter. This works, but leads to a lot of boilerplate, and means users don't benefit from the simple, unified API that content loaders offer. In most cases users tend to individually create loader libraries that they share between pages. + +This proposal introduces a new kind of loader that fetches data from an API at runtime, rather than build time. As with other content loaders, these loaders abstract the loading logic, meaning users don't need to understand the details of how data is loaded. These loaders can be distributed as node modules, or injected by integrations. + +# Goals + +- a new type of **live content loader** that is executed at runtime +- integration with user-facing `getEntry` and `getCollection` functions, allowing developers to use **a familiar, common API** to fetch data +- loader-specific **query and filters**, which a loader can define and pass to the API +- **type-safe** data and query options, defined by the loader as generic types +- support for user-defined **Zod schemas**, executed at runtime, to validate or transform the data returned by the loader. +- support for runtime **markdown rendering**, using a helper function provided in the loader context. +- optional **integration with [route caching](https://github.com/withastro/roadmap/issues/1140)**, allowing loaders to define cache tags and expiry times associated with the data which are then available to the user + +# Non-Goals + +- server-side caching of the data. Instead it would integrate with the route cache and HTTP caches to cache the full page response, or individual loaders could implement their own API caching. +- rendering of MDX or other content-like code. This isn't something that can be done at runtime. +- support for image processing, either in the Zod schema or Markdown. This is not something that can be done at runtime. +- loader-defined Zod schemas. Instead, loaders define types using TypeScript generics. Users can define their own Zod schemas to validate or transform the data returned by the loader, which Astro will execute at runtime. +- updating the content layer data store. Live loaders return data directly and do not update the store. +- support for existing loaders. They will have a different API. Developers could in theory use shared logic, but the loader API will be different + +# Detailed Design + +While the user-facing API is similar to the existing content loaders, the implementation is significantly different. + +## Loader API + +A live loader is an object with two methods: `loadCollection` and `loadEntry`. For libraries that distribute a loader, the convention for these will be for users to call a function that returns a loader object, which is then passed to the `defineCollection` function. This allows the user to pass in any configuration options they need. The loader object is then passed to the `defineCollection` function. + +The `loadCollection` and `loadEntry` methods are called when the user calls `getCollection` or `getEntry`. They return the requested data from the function, unlike existing loaders which are responsible for storing the data in the content layer data store. + +```ts +// storeloader.ts + +export function storeLoader({ field, key }): LiveLoader { + return { + name: "store-loader", + loadCollection: async ({ filter }) => { + // ... + return { + entries: products.map((product) => ({ + id: product.id, + data: product, + })), + }; + }, + loadEntry: async ({ filter }) => { + // ... + return { + id: filter.id, + data: product, + }; + }, + }; +} +``` + +## Loader execution + +Existing content loaders are executed at build time, and the data is stored in the content layer data store, which is then available during rendering. The new live loaders are executed at runtime, and the data is returned directly. + +The new `live.config.ts` file has similar syntax to the existing `content.config.ts` file, but it is compiled as part of the build process and included in the build so that it can be called at runtime. + +## Filters + +For existing collections, `getCollection` accepts an optional function to filter the collection. This filtering is performed in-memory on the data returned from the store. This is not an efficient approach for live loaders, which are likely to be making network requests for the data at request time. Loading all of the entries and then filtering them on the client would cause over-fetching, so it is preferable to filter the data natively in the API. + +For this reason, the `getCollection` and `getEntry` methods accept a query object, which is passed to the loader `loadEntry` and `loadCollection` functions. This is an arbitrary object, the type of which is defined by the loader. The loader can then use this filter to fetch the data from the API, according to the API's query syntax. The `getEntry` function also has a shorthand syntax for querying a single entry by ID by passing a string that matches the existing `getEntry` syntax. This is passed to the loader as an object with a single `id` property. + +## Type Safety + +The `LiveLoader` type is a generic type that takes three parameters: + +- `TData`: the type of the data returned by the loader +- `TEntryFilter`: the type of the filter object passed to `getEntry` +- `TCollectionFilter`: the type of the filter object passed to `getCollection` + +These types will be used to type the `loadCollection` and `loadEntry` methods. + +```ts +// storeloader.ts +import type { LiveLoader } from "astro/loaders"; +import { type Product, loadStoreData } from "./lib/api.ts"; + +interface StoreCollectionFilter { + category?: string; +} + +interface StoreEntryFilter { + slug?: string; +} + +export function storeLoader({ + field, + key, +}): LiveLoader { + return { + name: "store-loader", + // `filter` is typed as `StoreCollectionFilter` + loadCollection: async ({ filter }) => { + // ... + }, + // `filter` is typed as `StoreEntryFilter` + loadEntry: async ({ filter }) => { + // ... + }, + }; +} +``` + +The `LiveLoader` type is defined as follows: + +```ts +export interface LiveDataEntry< + TData extends Record = Record +> { + /** The ID of the entry. Unique per collection. */ + id: string; + /** The entry data */ + data: TData; + /** Optional cache hints */ + cache?: { + /** Cache tags */ + tags?: string[]; + /** Maximum age of the response in seconds */ + maxAge?: number; + }; +} + +export interface LiveDataCollection< + TData extends Record = Record +> { + entries: Array>; + /** Optional cache hints */ + cache?: { + /** Cache tags */ + tags?: string[]; + /** Maximum age of the response in seconds */ + maxAge?: number; + }; +} + +export interface LoadEntryContext { + filter: TEntryFilter extends never ? { id: string } : TEntryFilter; +} + +export interface LoadCollectionContext { + filter?: TCollectionFilter; +} + +export interface LiveLoader< + TData extends Record = Record, + TEntryFilter extends Record | never = never, + TCollectionFilter extends Record | never = never +> { + /** Unique name of the loader, e.g. the npm package name */ + name: string; + /** Load a single entry */ + loadEntry: ( + context: LoadEntryContext + ) => Promise | undefined>; + /** Load a collection of entries */ + loadCollection: ( + context: LoadCollectionContext + ) => Promise>; +} +``` + +The user-facing `getCollection` and `getEntry` methods exported from `astro:content` will also be typed with these types, so that the user can call them in a type-safe way. + +Users will still be able to define a Zod schema inside `defineCollection` to validate the data returned by the loader. If provided, this schema will also be used to infer the returned type of `getCollection` and `getEntry` for the collection, taking precedence over the loader type. This means that users can use the loader to fetch data from an API, and then use Zod to validate or transform the data before it is returned. + +## Caching + +The returned data is not cached by Astro, but a loader can provide hints to assist in caching the response. This would be designed to integrate with the proposed [route caching API](https://github.com/withastro/roadmap/issues/1140), but could also be used to manually set response headers. The scope of this RFC does not include details on the route cache integration, but will illustrate how the loader can provide hints that can then be used by the route cache or other caching mechanisms. + +Loader responses can include a `cache` object that contains the following properties: + +- `tags`: an array of strings that can be used to tag the response. This is useful for cache invalidation. +- `maxAge`: a number that specifies the maximum age of the response in seconds. This is useful for setting the cache expiry time. + +The loader does not define how these should be used, and the user is free to use them in any way they like. + +For example, a loader could return the following object for a collection: + +```ts +return { + entries: products.map((product) => ({ + id: product.id, + data: product, + })), + cache: { + tags: ["products", "clothes"], + maxAge: 60 * 60, // 1 hour + }, +}; +``` + +This would allow the user to tag the response with the `products` and `clothes` tags, and set the expiry time to 1 hour. The user could then use these tags to invalidate the cache when the data changes. + +The loader can also provide a `cache` object for an individual entry, allowing fine-grained cache control: + +```ts +return { + id: filter.id, + data: product, + cache: { + tags: ["products", "clothes", `product-${filter.id}`], + maxAge: 60 * 60, // 1 hour + }, +}; +``` + +When the user calls `getCollection` or `getEntry`, the response will include a cache object that contains the tags and expiry time. The user can then use this information to cache the response in their own caching layer, or pass it to the route cache. + +This example shows how cache tags and expiry time in the response headers: + +```astro +--- +import { getEntry } from "astro:content"; +import Product from "../components/Product.astro"; + +const product = await getEntry("products", Astro.params.id); + +Astro.response.headers.set("Cache-Tag", product.cache.tags.join(",")); +Astro.response.headers.set("CDN-Cache-Control", `s-maxage=${product.cache.maxAge}`); +--- + +``` + +# Testing Strategy + +Much of the testing strategy will be similar to the existing content loaders, as integration tests work in the same way. It will also be easier to test the loaders in isolation, as they are not dependent on the content layer data store. + +End-to-end tests will be added to test the bundling and runtime execution of the loaders. + +Type tests will be added to ensure that the generated types are correct, and that the user can call the `getCollection` and `getEntry` methods with the correct types. + +# Drawbacks + +- This is a significant addition to the content collections API, and will work to implement and document. +- This is a new API for loader developers to learn, and existing loaders cannot be trivially converted. While the API and mental model are simpler than the existing content loaders, it is still a new API that will require some work for developers to implement. +- Unlike the content layer APIs, there will not be any built-in loaders for this API, so it will be up to the community to implement them. There will be limited value to this featured until there are a number of loaders available. +- The user-facing API is similar but not identical to the existing content loaders, which may cause confusion for users. The behavior is also different, as the data is not stored in the content layer data store. This means that users will need to understand the difference between the two APIs. + +# Alternatives + +- **Do nothing**: For regularly-updated data they will need to use any APIs directly in the frontmatter. This is the current approach, and while it works, it is not ideal. It means that users need to implement their own loading logic. This tends to involve a lot of boilerplate, and there is no common API for accessing the data. +- **Add support for updating the content layer data store at runtime**: This would allow users to update the data in the content layer data store, for example via a webhook. This would be significantly more complex and would require a lot of work to implement. It would also require users provision third-party database services to support this in a serverless environment. + +# Adoption strategy + +- This would be released as an experimental feature, with a flag to enable it. +- As live collections are defined in a new file, existing sites will not be affected by this change unless they add new collections that use it. + +# Unresolved Questions + +- The proposed **name of the file** is potentially confusing. While the name `live.config.ts` is analogous to the existing `content.config.ts` file, neither are really configuration files, and have more in common with actions or middleware files. Would it better to not use `config` in the name, or would that be confusing when compared to the existing file? Possible alternatives: `src/live.ts`, `src/live-content.ts`, `src/live-content.config.ts`, `src/content.live.ts`... + +- The way to handle **rendering Markdown**, or even whether to support it. The most likely approach is to add a `renderMarkdown` helper function to the loader context that can be used to render Markdown to HTML, which would be stored in the `entry.rendered.html` field, similar to existing collections. This would allow use of the same `render()` function as existing collections. This would use a pre-configured instance of the renderer from the built-in `@astrojs/markdown-remark` package, using the user's Markdown config. This helper would be also likely be added to existing content layer loaders, as users have complained that is hard to do manually. This may be extending the scope too far, but it is a common use case for loaders. We may not want to encourage rendering Markdown inside loaders, as it could lead to performance issues. It may be confusing that image processing is unsupported, and nor is MDX. From 17be0572a9a79388340c7b9c01984b20f4d3dd7c Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Sat, 3 May 2025 10:56:38 +0100 Subject: [PATCH 02/14] Update types --- proposals/0055-live-content-loaders.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 3e84b5db..3d8a4b26 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -263,18 +263,20 @@ export interface LiveDataCollection< }; } -export interface LoadEntryContext { - filter: TEntryFilter extends never ? { id: string } : TEntryFilter; +export interface LoadEntryContext { + filter: TEntryFilter extends undefined + ? { + id: string; + } + : TEntryFilter; } - -export interface LoadCollectionContext { +export interface LoadCollectionContext { filter?: TCollectionFilter; } - export interface LiveLoader< TData extends Record = Record, - TEntryFilter extends Record | never = never, - TCollectionFilter extends Record | never = never + TEntryFilter extends Record | undefined = undefined, + TCollectionFilter extends Record | undefined = undefined > { /** Unique name of the loader, e.g. the npm package name */ name: string; From 32576164ef9bfe1c90575e7a3a12a956140424f2 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 8 May 2025 10:27:01 +0100 Subject: [PATCH 03/14] Rename cache hint object --- proposals/0055-live-content-loaders.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 3d8a4b26..2bf1ee20 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -242,7 +242,7 @@ export interface LiveDataEntry< /** The entry data */ data: TData; /** Optional cache hints */ - cache?: { + cacheHint?: { /** Cache tags */ tags?: string[]; /** Maximum age of the response in seconds */ @@ -255,7 +255,7 @@ export interface LiveDataCollection< > { entries: Array>; /** Optional cache hints */ - cache?: { + cacheHint?: { /** Cache tags */ tags?: string[]; /** Maximum age of the response in seconds */ @@ -299,7 +299,7 @@ Users will still be able to define a Zod schema inside `defineCollection` to val The returned data is not cached by Astro, but a loader can provide hints to assist in caching the response. This would be designed to integrate with the proposed [route caching API](https://github.com/withastro/roadmap/issues/1140), but could also be used to manually set response headers. The scope of this RFC does not include details on the route cache integration, but will illustrate how the loader can provide hints that can then be used by the route cache or other caching mechanisms. -Loader responses can include a `cache` object that contains the following properties: +Loader responses can include a `cacheHint` object that contains the following properties: - `tags`: an array of strings that can be used to tag the response. This is useful for cache invalidation. - `maxAge`: a number that specifies the maximum age of the response in seconds. This is useful for setting the cache expiry time. @@ -314,7 +314,7 @@ return { id: product.id, data: product, })), - cache: { + cacheHint: { tags: ["products", "clothes"], maxAge: 60 * 60, // 1 hour }, @@ -323,14 +323,14 @@ return { This would allow the user to tag the response with the `products` and `clothes` tags, and set the expiry time to 1 hour. The user could then use these tags to invalidate the cache when the data changes. -The loader can also provide a `cache` object for an individual entry, allowing fine-grained cache control: +The loader can also provide a `cacheHint` object for an individual entry, allowing fine-grained cache control: ```ts return { id: filter.id, data: product, - cache: { - tags: ["products", "clothes", `product-${filter.id}`], + cacheHint: { + tags: [`product-${filter.id}`], maxAge: 60 * 60, // 1 hour }, }; @@ -347,8 +347,8 @@ import Product from "../components/Product.astro"; const product = await getEntry("products", Astro.params.id); -Astro.response.headers.set("Cache-Tag", product.cache.tags.join(",")); -Astro.response.headers.set("CDN-Cache-Control", `s-maxage=${product.cache.maxAge}`); +Astro.response.headers.set("Cache-Tag", product.cacheHint.tags.join(",")); +Astro.response.headers.set("CDN-Cache-Control", `s-maxage=${product.cacheHint.maxAge}`); --- ``` From 4f201f9b6a7c5b3dfc2854bbd4dbe26181d153f1 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 23 May 2025 11:04:41 +0100 Subject: [PATCH 04/14] Update API with new function names and explicit error handling --- proposals/0055-live-content-loaders.md | 319 +++++++++++++++++++------ 1 file changed, 250 insertions(+), 69 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 2bf1ee20..0e8df08c 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -21,7 +21,7 @@ # Summary -Adds support for live data to content collections. Defines a new type of content loader that fetches data at runtime rather than build time, allowing users to get the data with a similar API. +Adds support for live data to content collections. Defines a new type of content loader that fetches data at runtime rather than build time, allowing users to get the data with dedicated functions that make the runtime behavior explicit. # Example @@ -47,33 +47,57 @@ export function storeLoader({ name: "store-loader", loadCollection: async ({ logger, filter }) => { logger.info(`Loading collection from ${field}`); - // load from API - const products = await loadStoreData({ field, key, filter }); - const entries = products.map((product) => ({ - id: product.id, - data: product, - })); - return { - entries, - }; + try { + // load from API + const products = await loadStoreData({ field, key, filter }); + const entries = products.map((product) => ({ + id: product.id, + data: product, + })); + return { + entries, + }; + } catch (error) { + logger.error(`Failed to load collection: ${error.message}`); + return { + error: { + message: `Failed to load products: ${error.message}`, + code: "COLLECTION_LOAD_ERROR", + }, + }; + } }, loadEntry: async ({ logger, filter }) => { logger.info(`Loading entry from ${field}`); - // load from API - const product = await loadStoreData({ - field, - key, - filter, - }); - - if (!product) { - logger.error(`Product not found`); - return; + try { + // load from API + const product = await loadStoreData({ + field, + key, + filter, + }); + + if (!product) { + return { + error: { + message: `Product not found`, + code: "ENTRY_NOT_FOUND", + }, + }; + } + return { + id: filter.id, + data: product, + }; + } catch (error) { + logger.error(`Failed to load entry: ${error.message}`); + return { + error: { + message: `Failed to load product: ${error.message}`, + code: "ENTRY_LOAD_ERROR", + }, + }; } - return { - id: filter.id, - data: product, - }; }, }; } @@ -95,23 +119,30 @@ const products = defineCollection({ export const collections = { products }; ``` -The loader can be used in the same way as a normal content collection: +The loader is accessed using new dedicated functions that make the runtime behavior explicit: ```astro --- -import { getCollection, getEntry } from "astro:content"; +import { getLiveCollection, getLiveEntry } from "astro:content"; -// Get all entries in a collection, like other collections -const allProducts = await getCollection("products"); +// Get all entries in a collection +const { data: allProducts, error } = await getLiveCollection("products"); +if (error) { + // Handle error gracefully + return Astro.redirect('/500'); +} // Live collections optionally allow extra filters to be passed in, defined by the loader -const clothes = await getCollection("products", { category: "clothes" }); +const { data: clothes } = await getLiveCollection("products", { category: "clothes" }); -// Get entrey by ID like other collections -const productById = await getEntry("products", Astro.params.id); +// Get entry by ID +const { data: productById } = await getLiveEntry("products", Astro.params.id); // Query a single entry using the object syntax -const productBySlug = await getEntry("products", { slug: Astro.params.slug }); +const { data: productBySlug, error: slugError } = await getLiveEntry("products", { slug: Astro.params.slug }); +if (slugError) { + return Astro.redirect('/404'); +} --- ``` @@ -123,14 +154,17 @@ This means that content collections are not suitable for pages that update frequ This proposal introduces a new kind of loader that fetches data from an API at runtime, rather than build time. As with other content loaders, these loaders abstract the loading logic, meaning users don't need to understand the details of how data is loaded. These loaders can be distributed as node modules, or injected by integrations. +The API uses dedicated functions (`getLiveCollection` and `getLiveEntry`) to make it explicit that these operations perform network requests at runtime, helping developers understand the performance implications and error handling requirements. + # Goals - a new type of **live content loader** that is executed at runtime -- integration with user-facing `getEntry` and `getCollection` functions, allowing developers to use **a familiar, common API** to fetch data +- dedicated user-facing functions `getLiveEntry` and `getLiveCollection` that make the **runtime behavior explicit** +- **built-in error handling** with consistent error response format - loader-specific **query and filters**, which a loader can define and pass to the API - **type-safe** data and query options, defined by the loader as generic types -- support for user-defined **Zod schemas**, executed at runtime, to validate or transform the data returned by the loader. -- support for runtime **markdown rendering**, using a helper function provided in the loader context. +- support for user-defined **Zod schemas**, executed at runtime, to validate or transform the data returned by the loader +- support for runtime **markdown rendering**, using a helper function provided in the loader context - optional **integration with [route caching](https://github.com/withastro/roadmap/issues/1140)**, allowing loaders to define cache tags and expiry times associated with the data which are then available to the user # Non-Goals @@ -144,13 +178,51 @@ This proposal introduces a new kind of loader that fetches data from an API at r # Detailed Design -While the user-facing API is similar to the existing content loaders, the implementation is significantly different. +The user-facing API uses dedicated functions `getLiveCollection` and `getLiveEntry` to make it clear these are runtime operations that may fail and have different performance characteristics than regular content collections. -## Loader API +## User-facing API + +The new functions are exported from `astro:content` alongside the existing functions: + +```ts +import { + getCollection, + getEntry, + getLiveCollection, + getLiveEntry, +} from "astro:content"; +``` + +These functions return a result object with either `data` or `error`, making error handling explicit and consistent: + +```ts +// Success case +const { data, error } = await getLiveCollection("products"); +if (error) { + // Handle error + console.error(error.message); + return Astro.redirect("/error"); +} +// Use data safely +data.forEach((product) => { + // ... +}); + +// With filters +const { data: electronics } = await getLiveCollection("products", { + category: "electronics", +}); + +// Single entry +const { data: product, error } = await getLiveEntry( + "products", + Astro.params.id +); +``` -A live loader is an object with two methods: `loadCollection` and `loadEntry`. For libraries that distribute a loader, the convention for these will be for users to call a function that returns a loader object, which is then passed to the `defineCollection` function. This allows the user to pass in any configuration options they need. The loader object is then passed to the `defineCollection` function. +## Loader API -The `loadCollection` and `loadEntry` methods are called when the user calls `getCollection` or `getEntry`. They return the requested data from the function, unlike existing loaders which are responsible for storing the data in the content layer data store. +A live loader is an object with two methods: `loadCollection` and `loadEntry`. These methods should handle errors gracefully and return either data or an error object: ```ts // storeloader.ts @@ -159,20 +231,48 @@ export function storeLoader({ field, key }): LiveLoader { return { name: "store-loader", loadCollection: async ({ filter }) => { - // ... - return { - entries: products.map((product) => ({ - id: product.id, - data: product, - })), - }; + try { + const products = await fetchProducts(filter); + return { + entries: products.map((product) => ({ + id: product.id, + data: product, + })), + }; + } catch (error) { + return { + error: { + message: `Failed to load products: ${error.message}`, + code: "COLLECTION_LOAD_ERROR", + cause: error, + }, + }; + } }, loadEntry: async ({ filter }) => { - // ... - return { - id: filter.id, - data: product, - }; + try { + const product = await fetchProduct(filter); + if (!product) { + return { + error: { + message: "Product not found", + code: "ENTRY_NOT_FOUND", + }, + }; + } + return { + id: filter.id, + data: product, + }; + } catch (error) { + return { + error: { + message: `Failed to load product: ${error.message}`, + code: "ENTRY_LOAD_ERROR", + cause: error, + }, + }; + } }, }; } @@ -188,15 +288,15 @@ The new `live.config.ts` file has similar syntax to the existing `content.config For existing collections, `getCollection` accepts an optional function to filter the collection. This filtering is performed in-memory on the data returned from the store. This is not an efficient approach for live loaders, which are likely to be making network requests for the data at request time. Loading all of the entries and then filtering them on the client would cause over-fetching, so it is preferable to filter the data natively in the API. -For this reason, the `getCollection` and `getEntry` methods accept a query object, which is passed to the loader `loadEntry` and `loadCollection` functions. This is an arbitrary object, the type of which is defined by the loader. The loader can then use this filter to fetch the data from the API, according to the API's query syntax. The `getEntry` function also has a shorthand syntax for querying a single entry by ID by passing a string that matches the existing `getEntry` syntax. This is passed to the loader as an object with a single `id` property. +For this reason, the `getLiveCollection` and `getLiveEntry` methods accept a query object, which is passed to the loader `loadEntry` and `loadCollection` functions. This is an arbitrary object, the type of which is defined by the loader. The loader can then use this filter to fetch the data from the API, according to the API's query syntax. The `getLiveEntry` function also has a shorthand syntax for querying a single entry by ID by passing a string that matches the existing `getEntry` syntax. This is passed to the loader as an object with a single `id` property. ## Type Safety The `LiveLoader` type is a generic type that takes three parameters: - `TData`: the type of the data returned by the loader -- `TEntryFilter`: the type of the filter object passed to `getEntry` -- `TCollectionFilter`: the type of the filter object passed to `getCollection` +- `TEntryFilter`: the type of the filter object passed to `getLiveEntry` +- `TCollectionFilter`: the type of the filter object passed to `getLiveCollection` These types will be used to type the `loadCollection` and `loadEntry` methods. @@ -234,6 +334,15 @@ export function storeLoader({ The `LiveLoader` type is defined as follows: ```ts +export interface LiveLoaderError { + /** Error message */ + message: string; + /** Error code for programmatic handling */ + code?: string; + /** Original error if applicable */ + cause?: unknown; +} + export interface LiveDataEntry< TData extends Record = Record > { @@ -263,6 +372,24 @@ export interface LiveDataCollection< }; } +export interface LiveDataEntryResult< + TData extends Record = Record +> { + data?: LiveDataEntry; + error?: LiveLoaderError; +} + +export interface LiveDataCollectionResult< + TData extends Record = Record +> { + data?: Array>; + error?: LiveLoaderError; + cacheHint?: { + tags?: string[]; + maxAge?: number; + }; +} + export interface LoadEntryContext { filter: TEntryFilter extends undefined ? { @@ -283,17 +410,62 @@ export interface LiveLoader< /** Load a single entry */ loadEntry: ( context: LoadEntryContext - ) => Promise | undefined>; + ) => Promise | { error: LiveLoaderError } | undefined>; /** Load a collection of entries */ loadCollection: ( context: LoadCollectionContext - ) => Promise>; + ) => Promise | { error: LiveLoaderError }>; } ``` -The user-facing `getCollection` and `getEntry` methods exported from `astro:content` will also be typed with these types, so that the user can call them in a type-safe way. +The user-facing `getLiveCollection` and `getLiveEntry` methods exported from `astro:content` will be typed to return the result format with separate `data` and `error` properties. + +Users will still be able to define a Zod schema inside `defineCollection` to validate the data returned by the loader. If provided, this schema will also be used to infer the returned type of `getLiveCollection` and `getLiveEntry` for the collection, taking precedence over the loader type. This means that users can use the loader to fetch data from an API, and then use Zod to validate or transform the data before it is returned. + +## Error Handling + +The consistent error handling approach provides several benefits: + +1. **Explicit error handling**: Developers must consider the error case +2. **Type safety**: TypeScript ensures error handling code is present +3. **Consistency**: All live loaders follow the same pattern +4. **Flexibility**: Loaders can provide custom error codes and messages + +Example error handling patterns: -Users will still be able to define a Zod schema inside `defineCollection` to validate the data returned by the loader. If provided, this schema will also be used to infer the returned type of `getCollection` and `getEntry` for the collection, taking precedence over the loader type. This means that users can use the loader to fetch data from an API, and then use Zod to validate or transform the data before it is returned. +```astro +--- +import { getLiveEntry, getLiveCollection } from "astro:content"; + +// Basic error handling +const { data: products, error } = await getLiveCollection("products"); +if (error) { + // Log for debugging + console.error(`Failed to load products: ${error.message}`); + + // Handle based on error code + if (error.code === 'RATE_LIMITED') { + return Astro.redirect('/too-many-requests'); + } + + // Generic error page + return Astro.redirect('/500'); +} + +// Fallback to cached data +const { data: liveData, error } = await getLiveEntry("products", id); +const product = liveData || getCachedProduct(id); + +// Custom error component +const { data, error } = await getLiveCollection("products"); +--- + +{error ? ( + +) : ( + +)} +``` ## Caching @@ -336,19 +508,23 @@ return { }; ``` -When the user calls `getCollection` or `getEntry`, the response will include a cache object that contains the tags and expiry time. The user can then use this information to cache the response in their own caching layer, or pass it to the route cache. - -This example shows how cache tags and expiry time in the response headers: +When the user calls `getLiveCollection` or `getLiveEntry`, the response will include cache hints that can be used. This example shows how to apply cache tags and expiry time in the response headers: ```astro --- -import { getEntry } from "astro:content"; +import { getLiveEntry } from "astro:content"; import Product from "../components/Product.astro"; -const product = await getEntry("products", Astro.params.id); +const { data: product, error, cacheHint } = await getLiveEntry("products", Astro.params.id); -Astro.response.headers.set("Cache-Tag", product.cacheHint.tags.join(",")); -Astro.response.headers.set("CDN-Cache-Control", `s-maxage=${product.cacheHint.maxAge}`); +if (error) { + return Astro.redirect('/404'); +} + +if (cacheHint) { + Astro.response.headers.set("Cache-Tag", cacheHint.tags.join(",")); + Astro.response.headers.set("CDN-Cache-Control", `s-maxage=${cacheHint.maxAge}`); +} --- ``` @@ -359,24 +535,29 @@ Much of the testing strategy will be similar to the existing content loaders, as End-to-end tests will be added to test the bundling and runtime execution of the loaders. -Type tests will be added to ensure that the generated types are correct, and that the user can call the `getCollection` and `getEntry` methods with the correct types. +Type tests will be added to ensure that the generated types are correct, and that the user can call the `getLiveCollection` and `getLiveEntry` methods with the correct types. + +Error handling tests will ensure that errors are properly propagated and that the result objects maintain type safety. # Drawbacks -- This is a significant addition to the content collections API, and will work to implement and document. +- This is a significant addition to the content collections API, and will require work to implement and document. - This is a new API for loader developers to learn, and existing loaders cannot be trivially converted. While the API and mental model are simpler than the existing content loaders, it is still a new API that will require some work for developers to implement. -- Unlike the content layer APIs, there will not be any built-in loaders for this API, so it will be up to the community to implement them. There will be limited value to this featured until there are a number of loaders available. -- The user-facing API is similar but not identical to the existing content loaders, which may cause confusion for users. The behavior is also different, as the data is not stored in the content layer data store. This means that users will need to understand the difference between the two APIs. +- Unlike the content layer APIs, there will not be any built-in loaders for this API, so it will be up to the community to implement them. There will be limited value to this feature until there are a number of loaders available. +- The dedicated functions (`getLiveCollection`/`getLiveEntry`) make it clear that these are different operations, but developers need to learn when to use which functions. # Alternatives - **Do nothing**: For regularly-updated data they will need to use any APIs directly in the frontmatter. This is the current approach, and while it works, it is not ideal. It means that users need to implement their own loading logic. This tends to involve a lot of boilerplate, and there is no common API for accessing the data. - **Add support for updating the content layer data store at runtime**: This would allow users to update the data in the content layer data store, for example via a webhook. This would be significantly more complex and would require a lot of work to implement. It would also require users provision third-party database services to support this in a serverless environment. +- **Use the same function names with different behavior**: This was the original proposal, but it could cause confusion about performance characteristics and error handling requirements. The dedicated functions make the runtime behavior explicit. # Adoption strategy - This would be released as an experimental feature, with a flag to enable it. - As live collections are defined in a new file, existing sites will not be affected by this change unless they add new collections that use it. +- Clear documentation will highlight the differences between `getCollection`/`getEntry` and `getLiveCollection`/`getLiveEntry`. +- Migration guides will help developers understand when to use each approach. # Unresolved Questions From ac115a8bf9a2c3fc0e2e8078e5cfb22c6ff6b955 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 23 May 2025 12:48:20 +0100 Subject: [PATCH 05/14] Rename from data to entries/entry --- proposals/0055-live-content-loaders.md | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 0e8df08c..6c42748d 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -126,20 +126,20 @@ The loader is accessed using new dedicated functions that make the runtime behav import { getLiveCollection, getLiveEntry } from "astro:content"; // Get all entries in a collection -const { data: allProducts, error } = await getLiveCollection("products"); +const { entries: allProducts, error } = await getLiveCollection("products"); if (error) { // Handle error gracefully return Astro.redirect('/500'); } // Live collections optionally allow extra filters to be passed in, defined by the loader -const { data: clothes } = await getLiveCollection("products", { category: "clothes" }); +const { entries: clothes } = await getLiveCollection("products", { category: "clothes" }); // Get entry by ID -const { data: productById } = await getLiveEntry("products", Astro.params.id); +const { entry: productById } = await getLiveEntry("products", Astro.params.id); // Query a single entry using the object syntax -const { data: productBySlug, error: slugError } = await getLiveEntry("products", { slug: Astro.params.slug }); +const { entry: productBySlug, error: slugError } = await getLiveEntry("products", { slug: Astro.params.slug }); if (slugError) { return Astro.redirect('/404'); } @@ -197,24 +197,24 @@ These functions return a result object with either `data` or `error`, making err ```ts // Success case -const { data, error } = await getLiveCollection("products"); +const { entries, error } = await getLiveCollection("products"); if (error) { // Handle error console.error(error.message); return Astro.redirect("/error"); } // Use data safely -data.forEach((product) => { +entries.forEach((product) => { // ... }); // With filters -const { data: electronics } = await getLiveCollection("products", { +const { entries: electronics } = await getLiveCollection("products", { category: "electronics", }); // Single entry -const { data: product, error } = await getLiveEntry( +const { entry: product, error } = await getLiveEntry( "products", Astro.params.id ); @@ -375,14 +375,14 @@ export interface LiveDataCollection< export interface LiveDataEntryResult< TData extends Record = Record > { - data?: LiveDataEntry; + entry?: LiveDataEntry; error?: LiveLoaderError; } export interface LiveDataCollectionResult< TData extends Record = Record > { - data?: Array>; + entries?: Array>; error?: LiveLoaderError; cacheHint?: { tags?: string[]; @@ -438,7 +438,7 @@ Example error handling patterns: import { getLiveEntry, getLiveCollection } from "astro:content"; // Basic error handling -const { data: products, error } = await getLiveCollection("products"); +const { entries: products, error } = await getLiveCollection("products"); if (error) { // Log for debugging console.error(`Failed to load products: ${error.message}`); @@ -452,18 +452,18 @@ if (error) { return Astro.redirect('/500'); } -// Fallback to cached data -const { data: liveData, error } = await getLiveEntry("products", id); +// Fallback to cached entries +const { entries: liveData, error } = await getLiveEntry("products", id); const product = liveData || getCachedProduct(id); // Custom error component -const { data, error } = await getLiveCollection("products"); +const { entries, error } = await getLiveCollection("products"); --- {error ? ( ) : ( - + )} ``` @@ -515,7 +515,7 @@ When the user calls `getLiveCollection` or `getLiveEntry`, the response will inc import { getLiveEntry } from "astro:content"; import Product from "../components/Product.astro"; -const { data: product, error, cacheHint } = await getLiveEntry("products", Astro.params.id); +const { entries: product, error, cacheHint } = await getLiveEntry("products", Astro.params.id); if (error) { return Astro.redirect('/404'); From 3630734346e42a4bb5daa627bef61346626e1195 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 23 May 2025 13:37:39 +0100 Subject: [PATCH 06/14] Use Error objects --- proposals/0055-live-content-loaders.md | 40 +++++++++----------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 6c42748d..e51c49b7 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -60,10 +60,9 @@ export function storeLoader({ } catch (error) { logger.error(`Failed to load collection: ${error.message}`); return { - error: { - message: `Failed to load products: ${error.message}`, - code: "COLLECTION_LOAD_ERROR", - }, + error: new Error(`Failed to load products: ${error.message}`, { + cause: error, + }), }; } }, @@ -79,10 +78,7 @@ export function storeLoader({ if (!product) { return { - error: { - message: `Product not found`, - code: "ENTRY_NOT_FOUND", - }, + error: new Error("Product not found"), }; } return { @@ -92,10 +88,9 @@ export function storeLoader({ } catch (error) { logger.error(`Failed to load entry: ${error.message}`); return { - error: { - message: `Failed to load product: ${error.message}`, - code: "ENTRY_LOAD_ERROR", - }, + error: new Error(`Failed to load product: ${error.message}`, { + cause: error, + }), }; } }, @@ -241,11 +236,9 @@ export function storeLoader({ field, key }): LiveLoader { }; } catch (error) { return { - error: { - message: `Failed to load products: ${error.message}`, - code: "COLLECTION_LOAD_ERROR", + error: new Error(`Failed to load products: ${error.message}`, { cause: error, - }, + }), }; } }, @@ -254,10 +247,7 @@ export function storeLoader({ field, key }): LiveLoader { const product = await fetchProduct(filter); if (!product) { return { - error: { - message: "Product not found", - code: "ENTRY_NOT_FOUND", - }, + error: new Error("Product not found"), }; } return { @@ -266,11 +256,9 @@ export function storeLoader({ field, key }): LiveLoader { }; } catch (error) { return { - error: { - message: `Failed to load product: ${error.message}`, - code: "ENTRY_LOAD_ERROR", + error: new Error(`Failed to load product: ${error.message}`, { cause: error, - }, + }), }; } }, @@ -410,11 +398,11 @@ export interface LiveLoader< /** Load a single entry */ loadEntry: ( context: LoadEntryContext - ) => Promise | { error: LiveLoaderError } | undefined>; + ) => Promise | { error: Error } | undefined>; /** Load a collection of entries */ loadCollection: ( context: LoadCollectionContext - ) => Promise | { error: LiveLoaderError }>; + ) => Promise | { error: Error }>; } ``` From 1776b2ba1f1527b062b57c9bd172189845abd7ae Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 23 May 2025 16:48:42 +0100 Subject: [PATCH 07/14] Update error handling --- proposals/0055-live-content-loaders.md | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index e51c49b7..0f62c4b7 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -392,17 +392,18 @@ export interface LiveLoader< TData extends Record = Record, TEntryFilter extends Record | undefined = undefined, TCollectionFilter extends Record | undefined = undefined + TError extends Error = Error > { /** Unique name of the loader, e.g. the npm package name */ name: string; /** Load a single entry */ loadEntry: ( context: LoadEntryContext - ) => Promise | { error: Error } | undefined>; + ) => Promise | { error: TError } | undefined>; /** Load a collection of entries */ loadCollection: ( context: LoadCollectionContext - ) => Promise | { error: Error }>; + ) => Promise | { error: TError }>; } ``` @@ -412,12 +413,7 @@ Users will still be able to define a Zod schema inside `defineCollection` to val ## Error Handling -The consistent error handling approach provides several benefits: - -1. **Explicit error handling**: Developers must consider the error case -2. **Type safety**: TypeScript ensures error handling code is present -3. **Consistency**: All live loaders follow the same pattern -4. **Flexibility**: Loaders can provide custom error codes and messages +Live loaders should handle errors gracefully and return an object with an `error` property. The error object should also include the original error if applicable, using the `cause` property. Example error handling patterns: @@ -427,11 +423,11 @@ import { getLiveEntry, getLiveCollection } from "astro:content"; // Basic error handling const { entries: products, error } = await getLiveCollection("products"); + if (error) { - // Log for debugging console.error(`Failed to load products: ${error.message}`); - // Handle based on error code + // Handle based on error code of custom error if (error.code === 'RATE_LIMITED') { return Astro.redirect('/too-many-requests'); } @@ -440,10 +436,6 @@ if (error) { return Astro.redirect('/500'); } -// Fallback to cached entries -const { entries: liveData, error } = await getLiveEntry("products", id); -const product = liveData || getCachedProduct(id); - // Custom error component const { entries, error } = await getLiveCollection("products"); --- From 6c78e7b710514107a46279dd374ef1d49f8cd209 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 23 May 2025 17:29:52 +0100 Subject: [PATCH 08/14] More error handling --- proposals/0055-live-content-loaders.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 0f62c4b7..eed24c3d 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -399,11 +399,11 @@ export interface LiveLoader< /** Load a single entry */ loadEntry: ( context: LoadEntryContext - ) => Promise | { error: TError } | undefined>; + ) => Promise | { error: TError | AstroError } | undefined>; /** Load a collection of entries */ loadCollection: ( context: LoadCollectionContext - ) => Promise | { error: TError }>; + ) => Promise | { error: TError | AstroError }>; } ``` @@ -421,9 +421,14 @@ Example error handling patterns: --- import { getLiveEntry, getLiveCollection } from "astro:content"; -// Basic error handling const { entries: products, error } = await getLiveCollection("products"); +// Use Astro's error handling +if (error) { + throw error; +} + +// Custom error handling if (error) { console.error(`Failed to load products: ${error.message}`); @@ -436,7 +441,9 @@ if (error) { return Astro.redirect('/500'); } -// Custom error component + + +// Display errors in the page const { entries, error } = await getLiveCollection("products"); --- From 7492f9b70d4481fd6478d4295d6742d964d8c411 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 28 May 2025 11:18:36 +0100 Subject: [PATCH 09/14] Update naming and correct custom error info --- proposals/0055-live-content-loaders.md | 28 +++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index eed24c3d..bbbd5157 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -21,7 +21,7 @@ # Summary -Adds support for live data to content collections. Defines a new type of content loader that fetches data at runtime rather than build time, allowing users to get the data with dedicated functions that make the runtime behavior explicit. +Adds support for live content collections, with a new type of loader that fetches data at runtime rather than build time. # Example @@ -280,11 +280,12 @@ For this reason, the `getLiveCollection` and `getLiveEntry` methods accept a que ## Type Safety -The `LiveLoader` type is a generic type that takes three parameters: +The `LiveLoader` type is a generic type that takes four parameters: - `TData`: the type of the data returned by the loader - `TEntryFilter`: the type of the filter object passed to `getLiveEntry` - `TCollectionFilter`: the type of the filter object passed to `getLiveCollection` +- `TError`: the type of the error returned by the loader These types will be used to type the `loadCollection` and `loadEntry` methods. @@ -301,10 +302,25 @@ interface StoreEntryFilter { slug?: string; } +class StoreLoaderError extends Error { + constructor(message: string, cause?: unknown) { + super(message); + this.name = "StoreLoaderError"; + if (cause) { + this.cause = cause; + } + } +} + export function storeLoader({ field, key, -}): LiveLoader { +}): LiveLoader< + Product, + StoreEntryFilter, + StoreCollectionFilter, + StoreLoaderError +> { return { name: "store-loader", // `filter` is typed as `StoreCollectionFilter` @@ -362,16 +378,18 @@ export interface LiveDataCollection< export interface LiveDataEntryResult< TData extends Record = Record + TError extends Error = Error > { entry?: LiveDataEntry; - error?: LiveLoaderError; + error?: TError; } export interface LiveDataCollectionResult< TData extends Record = Record + TError extends Error = Error > { entries?: Array>; - error?: LiveLoaderError; + error?: TError; cacheHint?: { tags?: string[]; maxAge?: number; From e0656979dfab6bd0a47b96de16116e0beb14faea Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 13 Jun 2025 11:53:51 +0100 Subject: [PATCH 10/14] Change to defineLiveCollection --- proposals/0055-live-content-loaders.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index bbbd5157..67df7858 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -102,11 +102,11 @@ A new `src/live.config.ts` file is introduced that uses the same syntax as the ` ```ts // src/live.config.ts -import { defineCollection } from "astro:content"; +import { defineLiveCollection } from "astro:content"; import { storeLoader } from "@mystore/astro-loader"; -const products = defineCollection({ +const products = defineLiveCollection({ type: "live", loader: storeLoader({ field: "products", key: process.env.STORE_KEY }), }); @@ -427,7 +427,7 @@ export interface LiveLoader< The user-facing `getLiveCollection` and `getLiveEntry` methods exported from `astro:content` will be typed to return the result format with separate `data` and `error` properties. -Users will still be able to define a Zod schema inside `defineCollection` to validate the data returned by the loader. If provided, this schema will also be used to infer the returned type of `getLiveCollection` and `getLiveEntry` for the collection, taking precedence over the loader type. This means that users can use the loader to fetch data from an API, and then use Zod to validate or transform the data before it is returned. +Users will still be able to define a Zod schema inside `defineLiveCollection` to validate the data returned by the loader. If provided, this schema will also be used to infer the returned type of `getLiveCollection` and `getLiveEntry` for the collection, taking precedence over the loader type. This means that users can use the loader to fetch data from an API, and then use Zod to validate or transform the data before it is returned. ## Error Handling From 9c637187fa638ffeb6788d99ca77d6677f83ba15 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 27 Jun 2025 13:07:46 +0100 Subject: [PATCH 11/14] Remove type: 'live' --- proposals/0055-live-content-loaders.md | 1 - 1 file changed, 1 deletion(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 67df7858..9981ceb7 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -107,7 +107,6 @@ import { defineLiveCollection } from "astro:content"; import { storeLoader } from "@mystore/astro-loader"; const products = defineLiveCollection({ - type: "live", loader: storeLoader({ field: "products", key: process.env.STORE_KEY }), }); From 02ce0925eb67c45969b2f23cb4d19e59f403d237 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 16 Oct 2025 11:24:06 +0100 Subject: [PATCH 12/14] Update to reflect changes in types --- proposals/0055-live-content-loaders.md | 190 ++++++++++++++++++++----- 1 file changed, 151 insertions(+), 39 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 9981ceb7..4e27c558 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -158,7 +158,7 @@ The API uses dedicated functions (`getLiveCollection` and `getLiveEntry`) to mak - loader-specific **query and filters**, which a loader can define and pass to the API - **type-safe** data and query options, defined by the loader as generic types - support for user-defined **Zod schemas**, executed at runtime, to validate or transform the data returned by the loader -- support for runtime **markdown rendering**, using a helper function provided in the loader context +- support for **rendered content** via a `rendered` property that loaders can return, allowing use of the `render()` function and `` component - optional **integration with [route caching](https://github.com/withastro/roadmap/issues/1140)**, allowing loaders to define cache tags and expiry times associated with the data which are then available to the user # Non-Goals @@ -167,6 +167,7 @@ The API uses dedicated functions (`getLiveCollection` and `getLiveEntry`) to mak - rendering of MDX or other content-like code. This isn't something that can be done at runtime. - support for image processing, either in the Zod schema or Markdown. This is not something that can be done at runtime. - loader-defined Zod schemas. Instead, loaders define types using TypeScript generics. Users can define their own Zod schemas to validate or transform the data returned by the loader, which Astro will execute at runtime. +- schema functions with `SchemaContext`. Live collections only support schema objects, not schema functions that receive context like `({ image }) => z.object({...})`. This is because runtime data fetching cannot use build-time asset processing utilities. - updating the content layer data store. Live loaders return data directly and do not update the store. - support for existing loaders. They will have a different API. Developers could in theory use shared logic, but the loader API will be different @@ -346,69 +347,68 @@ export interface LiveLoaderError { cause?: unknown; } +export interface CacheHint { + /** Cache tags */ + tags?: Array; + /** Last modified time of the content */ + lastModified?: Date; +} + export interface LiveDataEntry< - TData extends Record = Record + TData extends Record = Record > { /** The ID of the entry. Unique per collection. */ id: string; /** The entry data */ data: TData; - /** Optional cache hints */ - cacheHint?: { - /** Cache tags */ - tags?: string[]; - /** Maximum age of the response in seconds */ - maxAge?: number; + /** Optional rendered content */ + rendered?: { + html: string; }; + /** Optional cache hints */ + cacheHint?: CacheHint; } export interface LiveDataCollection< - TData extends Record = Record + TData extends Record = Record > { entries: Array>; /** Optional cache hints */ - cacheHint?: { - /** Cache tags */ - tags?: string[]; - /** Maximum age of the response in seconds */ - maxAge?: number; - }; + cacheHint?: CacheHint; } export interface LiveDataEntryResult< - TData extends Record = Record + TData extends Record = Record, TError extends Error = Error > { entry?: LiveDataEntry; - error?: TError; + error?: TError | LiveCollectionError; + cacheHint?: CacheHint; } export interface LiveDataCollectionResult< - TData extends Record = Record + TData extends Record = Record, TError extends Error = Error > { entries?: Array>; - error?: TError; - cacheHint?: { - tags?: string[]; - maxAge?: number; - }; + error?: TError | LiveCollectionError; + cacheHint?: CacheHint; } -export interface LoadEntryContext { - filter: TEntryFilter extends undefined +export interface LoadEntryContext { + filter: TEntryFilter extends never ? { id: string; } : TEntryFilter; } -export interface LoadCollectionContext { +export interface LoadCollectionContext { filter?: TCollectionFilter; } export interface LiveLoader< - TData extends Record = Record, - TEntryFilter extends Record | undefined = undefined, - TCollectionFilter extends Record | undefined = undefined + TData extends Record = Record, + TEntryFilter extends Record | never = never, + TCollectionFilter extends Record | never = never, TError extends Error = Error > { /** Unique name of the loader, e.g. the npm package name */ @@ -416,11 +416,11 @@ export interface LiveLoader< /** Load a single entry */ loadEntry: ( context: LoadEntryContext - ) => Promise | { error: TError | AstroError } | undefined>; + ) => Promise | undefined | { error: TError }>; /** Load a collection of entries */ loadCollection: ( context: LoadCollectionContext - ) => Promise | { error: TError | AstroError }>; + ) => Promise | { error: TError }>; } ``` @@ -428,10 +428,69 @@ The user-facing `getLiveCollection` and `getLiveEntry` methods exported from `as Users will still be able to define a Zod schema inside `defineLiveCollection` to validate the data returned by the loader. If provided, this schema will also be used to infer the returned type of `getLiveCollection` and `getLiveEntry` for the collection, taking precedence over the loader type. This means that users can use the loader to fetch data from an API, and then use Zod to validate or transform the data before it is returned. +### Schema Restrictions + +Unlike build-time content collections, live collections **do not support schema functions**. The schema must be a Zod schema object, not a function that receives the `image` helper: + +```ts +// Supported - Schema object +const products = defineLiveCollection({ + loader: storeLoader({ ... }), + schema: z.object({ + title: z.string(), + price: z.number(), + }), +}); + +// ❌ Not supported - Schema function +const products = defineLiveCollection({ + loader: storeLoader({ ... }), + schema: ({ image }) => z.object({ // Error: schema functions not allowed + title: z.string(), + cover: image(), + }), +}); +``` + +This restriction exists because the `image()` function is designed for build-time asset processing. Live collections fetch data at runtime, when build-time asset processing is not available. Image references in live collections should come pre-processed from the API or be handled client-side + ## Error Handling Live loaders should handle errors gracefully and return an object with an `error` property. The error object should also include the original error if applicable, using the `cause` property. +### Built-in Error Classes + +Astro provides built-in error classes for common live collection scenarios: + +- **`LiveCollectionError`**: Base error class for all live collection errors +- **`LiveEntryNotFoundError`**: Returned when an entry cannot be found +- **`LiveCollectionValidationError`**: Returned when data fails schema validation +- **`LiveCollectionCacheHintError`**: Returned when cache hints are invalid + +All error classes have a static `is()` method for type-safe error checking: + +```astro +--- +import { getLiveEntry } from "astro:content"; +import { LiveEntryNotFoundError } from "astro/loaders"; + +const { entry, error } = await getLiveEntry("products", Astro.params.id); + +if (LiveEntryNotFoundError.is(error)) { + console.error(`Product not found: ${error.message}`); + Astro.response.status = 404; + return Astro.redirect('/404'); +} + +if (error) { + console.error(`Unexpected error: ${error.message}`); + return Astro.redirect('/500'); +} +--- +``` + +### Error Handling Patterns + Example error handling patterns: ```astro @@ -473,12 +532,12 @@ const { entries, error } = await getLiveCollection("products"); ## Caching -The returned data is not cached by Astro, but a loader can provide hints to assist in caching the response. This would be designed to integrate with the proposed [route caching API](https://github.com/withastro/roadmap/issues/1140), but could also be used to manually set response headers. The scope of this RFC does not include details on the route cache integration, but will illustrate how the loader can provide hints that can then be used by the route cache or other caching mechanisms. +The returned data is not cached by Astro, but a loader can provide hints to assist in caching the response. This would be designed to integrate with the proposed [route caching API](https://github.com/withastro/roadmap/pull/1245), but could also be used to manually set response headers. The scope of this RFC does not include details on the route cache integration, but will illustrate how the loader can provide hints that can then be used by the route cache or other caching mechanisms. Loader responses can include a `cacheHint` object that contains the following properties: - `tags`: an array of strings that can be used to tag the response. This is useful for cache invalidation. -- `maxAge`: a number that specifies the maximum age of the response in seconds. This is useful for setting the cache expiry time. +- `lastModified`: a Date object that specifies when the content was last modified. This is useful for HTTP cache headers like `Last-Modified` and `If-Modified-Since`. The loader does not define how these should be used, and the user is free to use them in any way they like. @@ -492,12 +551,12 @@ return { })), cacheHint: { tags: ["products", "clothes"], - maxAge: 60 * 60, // 1 hour + lastModified: new Date(product.updatedAt), }, }; ``` -This would allow the user to tag the response with the `products` and `clothes` tags, and set the expiry time to 1 hour. The user could then use these tags to invalidate the cache when the data changes. +This would allow the user to tag the response with the `products` and `clothes` tags, and indicate when the content was last modified. The user could then use these tags to invalidate the cache when the data changes. The loader can also provide a `cacheHint` object for an individual entry, allowing fine-grained cache control: @@ -507,12 +566,12 @@ return { data: product, cacheHint: { tags: [`product-${filter.id}`], - maxAge: 60 * 60, // 1 hour + lastModified: new Date(product.updatedAt), }, }; ``` -When the user calls `getLiveCollection` or `getLiveEntry`, the response will include cache hints that can be used. This example shows how to apply cache tags and expiry time in the response headers: +When the user calls `getLiveCollection` or `getLiveEntry`, the response will include cache hints that can be used. This example shows how to apply cache tags and last modified time in the response headers: ```astro --- @@ -527,12 +586,65 @@ if (error) { if (cacheHint) { Astro.response.headers.set("Cache-Tag", cacheHint.tags.join(",")); - Astro.response.headers.set("CDN-Cache-Control", `s-maxage=${cacheHint.maxAge}`); + if (cacheHint.lastModified) { + Astro.response.headers.set("Last-Modified", cacheHint.lastModified.toUTCString()); + } } --- ``` +## Rendered Content + +Loaders can optionally return a `rendered` property containing HTML content. This allows entries to be rendered using the same `render()` function and `` component as build-time collections: + +```ts +// articleloader.ts +export function articleLoader(config: { apiKey: string }): LiveLoader
{ + return { + name: "article-loader", + loadEntry: async ({ filter }) => { + const article = await fetchFromCMS({ + apiKey: config.apiKey, + type: "article", + id: filter.id, + }); + + return { + id: article.id, + data: article, + rendered: { + // Assuming the CMS returns HTML content + html: article.htmlContent, + }, + }; + }, + // ... + }; +} +``` + +The rendered content can then be used in pages: + +```astro +--- +import { getLiveEntry, render } from "astro:content"; + +const { entry, error } = await getLiveEntry("articles", Astro.params.id); + +if (error) { + return Astro.rewrite('/404'); +} + +const { Content } = await render(entry); +--- + +

{entry.data.title}

+ +``` + +If a loader does not return a `rendered` property, the `` component will render nothing. + # Testing Strategy Much of the testing strategy will be similar to the existing content loaders, as integration tests work in the same way. It will also be easier to test the loaders in isolation, as they are not dependent on the content layer data store. @@ -567,4 +679,4 @@ Error handling tests will ensure that errors are properly propagated and that th - The proposed **name of the file** is potentially confusing. While the name `live.config.ts` is analogous to the existing `content.config.ts` file, neither are really configuration files, and have more in common with actions or middleware files. Would it better to not use `config` in the name, or would that be confusing when compared to the existing file? Possible alternatives: `src/live.ts`, `src/live-content.ts`, `src/live-content.config.ts`, `src/content.live.ts`... -- The way to handle **rendering Markdown**, or even whether to support it. The most likely approach is to add a `renderMarkdown` helper function to the loader context that can be used to render Markdown to HTML, which would be stored in the `entry.rendered.html` field, similar to existing collections. This would allow use of the same `render()` function as existing collections. This would use a pre-configured instance of the renderer from the built-in `@astrojs/markdown-remark` package, using the user's Markdown config. This helper would be also likely be added to existing content layer loaders, as users have complained that is hard to do manually. This may be extending the scope too far, but it is a common use case for loaders. We may not want to encourage rendering Markdown inside loaders, as it could lead to performance issues. It may be confusing that image processing is unsupported, and nor is MDX. +- Whether to provide a **`renderMarkdown` helper function** in the loader context. While the `rendered` property is supported and loaders can return pre-rendered HTML, there's a question about whether Astro should provide a helper to render Markdown at runtime within loaders. The most likely approach would be to add a `renderMarkdown(content: string)` helper function to the loader context that uses a pre-configured instance of the renderer from the built-in `@astrojs/markdown-remark` package. This helper could also be added to existing content layer loaders, as users have requested it. However, there are concerns: (1) it may encourage performance-intensive operations in loaders, (2) it may be confusing that image processing is unsupported, (3) it would not support MDX, and it would likely not be possible to use the user's configured Markdown options as these would not be serializable. Overall, while it could be a useful convenience, it may introduce more complexity and potential for misuse than the benefits it provides. From 85c14ad798d765d01117e92083dbac778cecaf7c Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 30 Jan 2026 14:16:10 +0000 Subject: [PATCH 13/14] Pass colelction name to loaders --- proposals/0055-live-content-loaders.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index 4e27c558..f0e4096f 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -45,7 +45,7 @@ export function storeLoader({ }): LiveLoader { return { name: "store-loader", - loadCollection: async ({ logger, filter }) => { + loadCollection: async ({ logger, filter, collection }) => { logger.info(`Loading collection from ${field}`); try { // load from API @@ -66,7 +66,7 @@ export function storeLoader({ }; } }, - loadEntry: async ({ logger, filter }) => { + loadEntry: async ({ logger, filter, collection }) => { logger.info(`Loading entry from ${field}`); try { // load from API @@ -324,11 +324,11 @@ export function storeLoader({ return { name: "store-loader", // `filter` is typed as `StoreCollectionFilter` - loadCollection: async ({ filter }) => { + loadCollection: async ({ filter, collection }) => { // ... }, // `filter` is typed as `StoreEntryFilter` - loadEntry: async ({ filter }) => { + loadEntry: async ({ filter, collection }) => { // ... }, }; @@ -401,9 +401,11 @@ export interface LoadEntryContext { id: string; } : TEntryFilter; + collection: string; } export interface LoadCollectionContext { filter?: TCollectionFilter; + collection: string; } export interface LiveLoader< TData extends Record = Record, From 2c00ce46ce97b60a748beb8d90e85febe6e9db4b Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 4 Feb 2026 11:45:04 +0000 Subject: [PATCH 14/14] Update proposals/0055-live-content-loaders.md Co-authored-by: Florian Lefebvre --- proposals/0055-live-content-loaders.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proposals/0055-live-content-loaders.md b/proposals/0055-live-content-loaders.md index f0e4096f..8048c800 100644 --- a/proposals/0055-live-content-loaders.md +++ b/proposals/0055-live-content-loaders.md @@ -17,7 +17,7 @@ - Implementation PR: https://github.com/withastro/astro/pull/13685 - Stage 1 Discussion: https://github.com/withastro/roadmap/discussions/1137 - Stage 2 Issue: https://github.com/withastro/roadmap/issues/1151 -- Stage 3 PR: +- Stage 3 PR: https://github.com/withastro/roadmap/pull/1164 # Summary