diff --git a/packages/astro/src/content/consts.ts b/packages/astro/src/content/consts.ts index b3aebada997c..4693c517dadd 100644 --- a/packages/astro/src/content/consts.ts +++ b/packages/astro/src/content/consts.ts @@ -35,7 +35,11 @@ export const CONTENT_FLAGS = [ ] as const; export const CONTENT_TYPES_FILE = 'content.d.ts'; + export const DATA_STORE_FILE = 'data-store.json'; +export const DATA_STORE_MANIFEST_FILE = '__manifest.json'; +export const DATA_STORE_DIR = 'data-store/'; + export const ASSET_IMPORTS_FILE = 'content-assets.mjs'; export const MODULES_IMPORTS_FILE = 'content-modules.mjs'; export const COLLECTIONS_MANIFEST_FILE = 'collections/collections.json'; diff --git a/packages/astro/src/content/content-layer.ts b/packages/astro/src/content/content-layer.ts index fbce711833f0..6554cf1a08af 100644 --- a/packages/astro/src/content/content-layer.ts +++ b/packages/astro/src/content/content-layer.ts @@ -16,6 +16,7 @@ import { ASSET_IMPORTS_FILE, COLLECTIONS_MANIFEST_FILE, CONTENT_LAYER_TYPE, + DATA_STORE_DIR, DATA_STORE_FILE, MODULES_IMPORTS_FILE, } from './consts.js'; @@ -463,6 +464,7 @@ async function simpleLoader( ), }); } + /** * Get the path to the data store file. * During development, this is in the `.astro` directory so that the Vite watcher can see it. @@ -471,3 +473,30 @@ async function simpleLoader( export function getDataStoreFile(settings: AstroSettings, isDev: boolean) { return new URL(DATA_STORE_FILE, isDev ? settings.dotAstroDir : settings.config.cacheDir); } + +/** + * Get the path to the data store directory. + * During development, this is in the `.astro` directory so that the Vite watcher can see it. + * In production, it's in the cache directory so that it's preserved between builds. + */ +export function getDataStoreDir(settings: AstroSettings, isDev: boolean) { + return new URL(DATA_STORE_DIR, isDev ? settings.dotAstroDir : settings.config.cacheDir); +} + +function contentLayerSingleton() { + let instance: ContentLayer | null = null; + return { + init: (options: ContentLayerOptions) => { + instance?.dispose(); + instance = new ContentLayer(options); + return instance; + }, + get: () => instance, + dispose: () => { + instance?.dispose(); + instance = null; + }, + }; +} + +export const globalContentLayer = contentLayerSingleton(); diff --git a/packages/astro/src/content/data-store.ts b/packages/astro/src/content/data-store.ts index 1e89919feacb..057a4112b26d 100644 --- a/packages/astro/src/content/data-store.ts +++ b/packages/astro/src/content/data-store.ts @@ -83,6 +83,39 @@ export class ImmutableDataStore { return this._collections; } + /** + * Converts an expanded manifest object to a collections map. + * + * Expanded manifest has file names swapped with actual file contents, + * in a form of either ESM imports or raw strings. + */ + static async manifestToMap(manifest: Record) { + const map = new Map(); + for (const [collectionName, chunks] of Object.entries(manifest)) { + const collection = new Map(); + for (const chunk of chunks) { + // Combine all string parts into a single string + let stringified = ''; + for (const data of chunk) { + // Handle strings and ESM default imports + stringified += typeof data === 'string' ? data : data.default; + } + + // Restore the collection chunk (up to 1000 entries) + const entries: Map = devalue.parse(stringified); + + // Combine into the full collection + for (const [id, entry] of entries) { + collection.set(id, entry); + } + } + + map.set(collectionName, collection); + } + + return map; + } + /** * Attempts to load a DataStore from the virtual module. * This only works in Vite. @@ -94,7 +127,11 @@ export class ImmutableDataStore { if (data.default instanceof Map) { return ImmutableDataStore.fromMap(data.default); } - const map = devalue.unflatten(data.default); + if (Array.isArray(data.default)) { + const map = devalue.unflatten(data.default); + return ImmutableDataStore.fromMap(map); + } + const map = await this.manifestToMap(data.default); return ImmutableDataStore.fromMap(map); } catch {} return new ImmutableDataStore(); diff --git a/packages/astro/src/content/mutable-data-store.ts b/packages/astro/src/content/mutable-data-store.ts index 9459645c29a8..bc867095e59b 100644 --- a/packages/astro/src/content/mutable-data-store.ts +++ b/packages/astro/src/content/mutable-data-store.ts @@ -1,15 +1,18 @@ import { existsSync, promises as fs, type PathLike } from 'node:fs'; import * as devalue from 'devalue'; import { Traverse } from 'neotraverse/modern'; +import type { XXHashAPI } from 'xxhash-wasm'; +import xxhash from 'xxhash-wasm'; import { imageSrcToImportId, importIdToSymbolName } from '../assets/utils/resolveImports.js'; import { AstroError, AstroErrorData } from '../core/errors/index.js'; -import { IMAGE_IMPORT_PREFIX } from './consts.js'; +import { emptyDir } from '../core/fs/index.js'; +import { DATA_STORE_MANIFEST_FILE, IMAGE_IMPORT_PREFIX } from './consts.js'; import { type DataEntry, ImmutableDataStore } from './data-store.js'; -import { contentModuleToId } from './utils.js'; +import { chunkMap, chunkString, contentModuleToId, sanitizeFileName } from './utils.js'; const SAVE_DEBOUNCE_MS = 500; - const MAX_DEPTH = 10; +const CHUNK_SIZE_LIMIT = 20 * 1024 * 1024; // 20MB in bytes /** * Extends the DataStore with the ability to change entries and write them to disk. @@ -17,6 +20,8 @@ const MAX_DEPTH = 10; */ export class MutableDataStore extends ImmutableDataStore { #file?: PathLike; + #manifestFile?: URL; + #dir?: URL; #assetsFile?: PathLike; #modulesFile?: PathLike; @@ -38,6 +43,9 @@ export class MutableDataStore extends ImmutableDataStore { #writeInProgress = false; #writeQueued = false; + #hasher?: XXHashAPI; + #chunking = false; + set(collectionName: string, key: string, value: unknown) { const collection = this._collections.get(collectionName) ?? new Map(); collection.set(String(key), value); @@ -254,7 +262,7 @@ export default new Map([\n${lines.join(',\n')}]); clearTimeout(this.#saveTimeout); } this.#saveTimeout = undefined; - if (this.#file) { + if (this.#file || (this.#manifestFile && this.#dir)) { await this.writeToDisk(); } this.#maybeResolveSavePromise(); @@ -274,7 +282,7 @@ export default new Map([\n${lines.join(',\n')}]); this.#saveTimeout = setTimeout(async () => { this.#saveTimeout = undefined; - if (this.#file) { + if (this.#file || (this.#manifestFile && this.#dir)) { await this.writeToDisk(); } this.#maybeResolveSavePromise(); @@ -436,10 +444,7 @@ export default new Map([\n${lines.join(',\n')}]); return devalue.stringify(sorted); } - async writeToDisk() { - if (!this.#dirty) { - return; - } + async writeToFile() { if (!this.#file) { throw new AstroError(AstroErrorData.UnknownFilesystemError); } @@ -468,6 +473,68 @@ export default new Map([\n${lines.join(',\n')}]); } } + async writeToDir() { + if (!this.#manifestFile || !this.#dir) { + throw new AstroError(AstroErrorData.UnknownFilesystemError); + } + if (!this.#hasher) { + this.#hasher = await xxhash(); + } + + try { + // Mark as clean before writing to disk so that it catches any changes that happen during the write + this.#dirty = false; + + // Keep track of written files to remove old ones + const writtenFiles = new Set(); + + const manifest: Record = {}; + + // Split by collection + for (const [collectionName, entries] of this._collections) { + manifest[collectionName] = []; + + // Split into chunks of 1000 entries each (avoid huge strings) + const chunkedCollection = chunkMap(entries, 1000); + for (const chunkedEntries of chunkedCollection) { + const stringified = devalue.stringify(chunkedEntries); + + // Further split string into chunks of <20MB each (avoid platform-specific single file size limits) + const chunkedStrings = chunkString(stringified, CHUNK_SIZE_LIMIT); + const parts = []; + for (const chunk of chunkedStrings) { + const fileName = `${sanitizeFileName(collectionName)}.${this.#hasher.h64ToString(chunk)}.json`; + await this.#writeFileAtomic(new URL(`./${fileName}`, this.#dir), chunk); + parts.push(fileName); + writtenFiles.add(fileName); + } + manifest[collectionName].push(parts); + } + } + + // Finally, write the manifest + await this.#writeFileAtomic(this.#manifestFile, JSON.stringify(manifest)); + writtenFiles.add(DATA_STORE_MANIFEST_FILE); + + // Remove any files that are no longer referenced in the manifest + emptyDir(this.#dir, writtenFiles); + } catch (err) { + throw new AstroError(AstroErrorData.UnknownFilesystemError, { cause: err }); + } + } + + async writeToDisk() { + if (!this.#dirty) { + return; + } + + if (this.#chunking) { + return this.writeToDir(); + } else { + return this.writeToFile(); + } + } + /** * Attempts to load a MutableDataStore from the virtual module. * This only works in Vite. @@ -476,7 +543,14 @@ export default new Map([\n${lines.join(',\n')}]); try { // @ts-expect-error - this is a virtual module const data = await import('astro:data-layer-content'); - const map = devalue.unflatten(data.default); + if (data.default instanceof Map) { + return MutableDataStore.fromMap(data.default); + } + if (Array.isArray(data.default)) { + const map = devalue.unflatten(data.default); + return MutableDataStore.fromMap(map); + } + const map = await this.manifestToMap(data.default); return MutableDataStore.fromMap(map); } catch {} return new MutableDataStore(); @@ -508,6 +582,48 @@ export default new Map([\n${lines.join(',\n')}]); store.#file = filePath; return store; } + + static async fromDir(dirPath: URL) { + const manifestPath = new URL(`./${DATA_STORE_MANIFEST_FILE}`, dirPath); + try { + if (existsSync(dirPath) && existsSync(manifestPath)) { + const data = await fs.readFile(manifestPath, 'utf-8'); + const manifest: Record = JSON.parse(data); + + if (manifest) { + // Read each file in the manifest + const parsed: Record = {}; + + for (const collection in manifest) { + parsed[collection] = []; + for (const chunks of manifest[collection]) { + parsed[collection].push( + await Promise.all( + chunks.map( + async (file) => await fs.readFile(new URL('./' + file, dirPath), 'utf-8'), + ), + ), + ); + } + } + + const map = await this.manifestToMap(parsed); + const store = await MutableDataStore.fromMap(map); + store.#manifestFile = manifestPath; + store.#dir = dirPath; + store.#chunking = true; + return store; + } + } else { + await fs.mkdir(dirPath, { recursive: true }); + } + } catch {} + const store = new MutableDataStore(); + store.#manifestFile = manifestPath; + store.#dir = dirPath; + store.#chunking = true; + return store; + } } // This is the scoped store for a single collection. It's a subset of the MutableDataStore API, and is the only public type. diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 6ac7da29f0a9..8783aa0bde92 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -896,3 +896,60 @@ export function safeStringify(value: unknown) { const seen = new WeakSet(); return JSON.stringify(value, safeStringifyReplacer(seen)); } + +const safeFileNameReplacers = [ + /[/?<>\\:*|"]/g, // common illegal characters + // eslint-disable-next-line no-control-regex + /[\x00-\x1f\x80-\x9f]/g, // unicode control codes + /^\.+$/, // unix reserved + /^(con|prn|aux|nul|com\d|lpt\d)(\..*)?$/i, // windows reserved +]; + +/** + * Cross-platform string sanitizer for file names + * Adapted from https://gist.github.com/barbietunnie/7bc6d48a424446c44ff4 + */ +export function sanitizeFileName(fileName: string, replacement = '_') { + let sanitized = fileName; + + for (const re of safeFileNameReplacers) { + sanitized = sanitized.replace(re, replacement); + } + + // truncate to 200 chars (leave space for hash and extension) + const encoded = new TextEncoder().encode(sanitized); + const truncated = encoded.slice(0, 200); + return new TextDecoder().decode(truncated); +} + +// Splits a string into chunks that are each below the specified byte size limit +export function chunkString(str: string, maxBytes: number): string[] { + const maxChars = Math.floor(maxBytes / 2); // assume average-case 2 bytes per char + const chunks = []; + + for (let i = 0; i < str.length; i += maxChars) { + chunks.push(str.slice(i, i + maxChars)); + } + + return chunks; +} + +// Splits a Map into equally sized chunks of Maps +export function chunkMap(map: Map, chunkSize: number): Map[] { + const chunks: Map[] = []; + let currentChunk = new Map(); + + for (const [key, value] of map) { + currentChunk.set(key, value); + if (currentChunk.size >= chunkSize) { + chunks.push(currentChunk); + currentChunk = new Map(); + } + } + + if (currentChunk.size > 0) { + chunks.push(currentChunk); + } + + return chunks; +} diff --git a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts index a70214378964..3973170eb61d 100644 --- a/packages/astro/src/content/vite-plugin-content-virtual-mod.ts +++ b/packages/astro/src/content/vite-plugin-content-virtual-mod.ts @@ -15,6 +15,7 @@ import { ASSET_IMPORTS_VIRTUAL_ID, CONTENT_MODULE_FLAG, CONTENT_RENDER_FLAG, + DATA_STORE_MANIFEST_FILE, DATA_STORE_VIRTUAL_ID, MODULES_IMPORTS_FILE, MODULES_MJS_ID, @@ -23,7 +24,7 @@ import { RESOLVED_VIRTUAL_MODULE_ID, VIRTUAL_MODULE_ID, } from './consts.js'; -import { getDataStoreFile } from './content-layer.js'; +import { getDataStoreDir, getDataStoreFile } from './content-layer.js'; import { getContentPaths, isDeferredModule } from './utils.js'; interface AstroContentVirtualModPluginParams { @@ -52,6 +53,7 @@ export function astroContentVirtualModPlugin({ settings, fs, }: AstroContentVirtualModPluginParams): Plugin { + let dataStoreDir: URL; let dataStoreFile: URL; let devServer: ViteDevServer; let liveConfig: string; @@ -59,7 +61,13 @@ export function astroContentVirtualModPlugin({ name: 'astro-content-virtual-mod-plugin', enforce: 'pre', config(_, env) { - dataStoreFile = getDataStoreFile(settings, env.command === 'serve'); + if (settings.config.experimental.dataStoreChunking) { + dataStoreDir = getDataStoreDir(settings, env.command === 'serve'); + dataStoreFile = new URL(DATA_STORE_MANIFEST_FILE, dataStoreDir); + } else { + dataStoreFile = getDataStoreFile(settings, env.command === 'serve'); + } + const contentPaths = getContentPaths( settings.config, undefined, @@ -159,6 +167,36 @@ export function astroContentVirtualModPlugin({ } const jsonData = await fs.promises.readFile(dataStoreFile, 'utf-8'); + if (settings.config.experimental.dataStoreChunking) { + try { + const manifest: Record = JSON.parse(jsonData); + const parsed: Record = {}; + + /** + * Convert manifest paths to imports to keep content files separated. + */ + for (const collection in manifest) { + parsed[collection] = manifest[collection].map((files) => + files.map( + (file) => + `@@IMPORT@@${rootRelativePath(settings.config.root, new URL('./' + file, dataStoreDir))}@@/IMPORT@@`, + ), + ); + } + + const code = dataToEsm(parsed, { + compact: true, + }).replace(/"@@IMPORT@@(.+?)@@\/IMPORT@@"/g, '(await import("$1?raw"))'); + + return { + code, + map: { mappings: '' }, + }; + } catch (err) { + const message = 'Could not parse data store manifest JSON file'; + this.error({ message, id, cause: err }); + } + } else { try { const parsed = JSON.parse(jsonData); return { @@ -172,6 +210,7 @@ export function astroContentVirtualModPlugin({ this.error({ message, id, cause: err }); } } + } if (id === ASSET_IMPORTS_RESOLVED_STUB_ID) { const assetImportsFile = new URL(ASSET_IMPORTS_FILE, settings.dotAstroDir); diff --git a/packages/astro/src/core/config/schemas/base.ts b/packages/astro/src/core/config/schemas/base.ts index f9cc154f3f1e..2ec74c12d21f 100644 --- a/packages/astro/src/core/config/schemas/base.ts +++ b/packages/astro/src/core/config/schemas/base.ts @@ -114,6 +114,7 @@ export const ASTRO_CONFIG_DEFAULTS = { queuedRendering: { enabled: false, }, + dataStoreChunking: false, }, } satisfies AstroUserConfig & { server: { open: boolean } }; @@ -548,6 +549,10 @@ export const AstroConfigSchema = z.object({ }) .optional() .prefault(ASTRO_CONFIG_DEFAULTS.experimental.queuedRendering), + dataStoreChunking: z + .boolean() + .optional() + .default(ASTRO_CONFIG_DEFAULTS.experimental.dataStoreChunking), }) .prefault({}), legacy: z diff --git a/packages/astro/src/core/dev/dev.ts b/packages/astro/src/core/dev/dev.ts index cd5041a0b353..f454e4453577 100644 --- a/packages/astro/src/core/dev/dev.ts +++ b/packages/astro/src/core/dev/dev.ts @@ -6,8 +6,11 @@ import { performance } from 'node:perf_hooks'; import colors from 'piccolore'; import { gt, major, minor, patch } from 'semver'; import type * as vite from 'vite'; -import { getDataStoreFile } from '../../content/content-layer.js'; -import { globalContentLayer } from '../../content/instance.js'; +import { + getDataStoreDir, + getDataStoreFile, + globalContentLayer, +} from '../../content/content-layer.js'; import { attachContentServerListeners } from '../../content/index.js'; import { MutableDataStore } from '../../content/mutable-data-store.js'; import { globalContentConfigObserver } from '../../content/utils.js'; @@ -104,8 +107,14 @@ export default async function dev(inlineConfig: AstroInlineConfig): Promise { +import { describeCases, loadFixture } from './test-utils.js'; + +const cases = [ + ['', { config: {}, dataStoreFileName: 'data-store.json' }], + [ + 'data store chunking enabled', + { + config: { experimental: { dataStoreChunking: true } }, + dataStoreFileName: 'data-store/__manifest.json', + }, + ], +]; + +describeCases('--mode', cases, (testCase) => { /** @type {import('./test-utils.js').Fixture} */ let fixture; let devDataStoreFile; @@ -17,10 +28,14 @@ describe('--mode', () => { before(async () => { fixture = await loadFixture({ + ...testCase.config, root: './fixtures/astro-mode/', }); - devDataStoreFile = new URL('./.astro/data-store.json', fixture.config.root); - prodDataStoreFile = new URL('./node_modules/.astro/data-store.json', fixture.config.root); + devDataStoreFile = new URL(`./.astro/${testCase.dataStoreFileName}`, fixture.config.root); + prodDataStoreFile = new URL( + `./node_modules/.astro/${testCase.dataStoreFileName}`, + fixture.config.root, + ); }); afterEach(() => { diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index 6a6930a7f9e5..4567867338a4 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -2,6 +2,7 @@ import { spawn } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { describe } from 'node:test'; import { fileURLToPath } from 'node:url'; import { glob } from 'tinyglobby'; import { Agent } from 'undici'; @@ -185,17 +186,42 @@ export async function loadFixture(inlineConfig) { } const dataStoreFile = path.join(root, '.astro', 'data-store.json'); + const dataStoreDir = path.join(root, '.astro', 'data-store'); return new Promise((resolve, reject) => { + let debounceTimer = null; + let timeoutTimer = null; + + const cleanup = () => { + if (debounceTimer) { + clearTimeout(debounceTimer); + debounceTimer = null; + } + if (timeoutTimer) { + clearTimeout(timeoutTimer); + timeoutTimer = null; + } + devServer.watcher.removeListener('change', changeHandler); + }; + const changeHandler = (fileName) => { - if (fileName === dataStoreFile) { - devServer.watcher.removeListener('change', changeHandler); - resolve(); + // Check if the changed file is in the data store directory or file + if (fileName === dataStoreFile || fileName.startsWith(dataStoreDir)) { + if (debounceTimer) { + clearTimeout(debounceTimer); + } + + debounceTimer = setTimeout(() => { + cleanup(); + resolve(); + }, 100); // 100ms debounce } }; + devServer.watcher.on('change', changeHandler); - setTimeout(() => { - devServer.watcher.removeListener('change', changeHandler); + + timeoutTimer = setTimeout(() => { + cleanup(); reject(new Error('Data store did not update within timeout')); }, timeout); }); @@ -372,3 +398,11 @@ export async function* streamAsyncIterator(stream) { reader.releaseLock(); } } + +export function describeCases(name, cases, fn) { + for (const [caseName, caseParams] of cases) { + describe(name + (caseName ? ` (${caseName})` : ''), () => { + fn(caseParams); + }); + } +} diff --git a/packages/astro/test/units/content-layer/store-persistence.test.ts b/packages/astro/test/units/content-layer/store-persistence.test.ts index 2f6caa3ddd7f..98a6ed07af47 100644 --- a/packages/astro/test/units/content-layer/store-persistence.test.ts +++ b/packages/astro/test/units/content-layer/store-persistence.test.ts @@ -5,209 +5,273 @@ import fs from 'node:fs/promises'; import { MutableDataStore } from '../../../dist/content/mutable-data-store.js'; import { createTempDir } from './test-helpers.ts'; -describe('Content Layer - Store Persistence', () => { - it('updates the store on new builds', async () => { - const tempDir = createTempDir(); - const dataStoreFile = new URL('./data-store.json', tempDir); - - // First build - create initial data - const store1 = new MutableDataStore(); - store1.set('dogs', 'beagle', { - id: 'beagle', - data: { breed: 'Beagle', temperament: ['Friendly'] }, +/** + * Persistence helpers for the two store modes: + * - file: single data-store.json + * - dir: chunked data-store/ directory (experimental dataStoreChunking) + */ +interface PersistenceMode { + label: string; + /** Save a store to disk */ + save: (store: MutableDataStore, dir: URL) => Promise; + /** Load a store from disk */ + load: (dir: URL) => Promise; +} + +const modes: PersistenceMode[] = [ + { + label: '', + async save(store, dir) { + const file = new URL('./data-store.json', dir); + await fs.writeFile(file, store.toString()); + }, + async load(dir) { + const file = new URL('./data-store.json', dir); + return MutableDataStore.fromFile(fileURLToPath(file)); + }, + }, + { + label: 'chunking enabled', + async save(store, _dir) { + // fromDir sets up the internal state for chunked writes; + // writeToDisk dispatches to writeToDir when chunking is active. + await store.writeToDisk(); + }, + async load(dir) { + const storeDir = new URL('./data-store/', dir); + return MutableDataStore.fromDir(storeDir); + }, + }, +]; + +for (const mode of modes) { + const suiteName = 'Content Layer - Store Persistence' + (mode.label ? ` (${mode.label})` : ''); + + describe(suiteName, () => { + it('updates the store on new builds', async () => { + const tempDir = createTempDir(); + + // First build - create initial data + let store1: MutableDataStore; + if (mode.label) { + // For chunked mode, load from dir to initialize chunking state + store1 = await mode.load(tempDir); + } else { + store1 = new MutableDataStore(); + } + store1.set('dogs', 'beagle', { + id: 'beagle', + data: { breed: 'Beagle', temperament: ['Friendly'] }, + }); + + // Save to disk + await mode.save(store1, tempDir); + + // Second build - load from disk and update + const store2 = await mode.load(tempDir); + + // Verify existing data persists + const beagle = store2.get('dogs', 'beagle'); + assert.ok(beagle); + assert.equal(beagle.data.breed, 'Beagle'); + + // Add new data + store2.set('dogs', 'poodle', { + id: 'poodle', + data: { breed: 'Poodle', temperament: ['Intelligent'] }, + }); + + // Save again + await mode.save(store2, tempDir); + + // Third build - verify both entries exist + const store3 = await mode.load(tempDir); + assert.equal(store3.values('dogs').length, 2); + assert.ok(store3.get('dogs', 'beagle')); + assert.ok(store3.get('dogs', 'poodle')); }); - // Save to disk - await fs.writeFile(dataStoreFile, store1.toString()); + it('clears the store on new build with force flag', async () => { + const tempDir = createTempDir(); - // Second build - load from disk and update - const store2 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); + // First build - create data + let store1: MutableDataStore; + if (mode.label) { + store1 = await mode.load(tempDir); + } else { + store1 = new MutableDataStore(); + } + store1.set('dogs', 'beagle', { + id: 'beagle', + data: { breed: 'Beagle' }, + }); + store1.metaStore().set('content-config-digest', 'digest1'); - // Verify existing data persists - const beagle = store2.get('dogs', 'beagle'); - assert.ok(beagle); - assert.equal(beagle.data.breed, 'Beagle'); + await mode.save(store1, tempDir); - // Add new data - store2.set('dogs', 'poodle', { - id: 'poodle', - data: { breed: 'Poodle', temperament: ['Intelligent'] }, - }); - - // Save again - await fs.writeFile(dataStoreFile, store2.toString()); - - // Third build - verify both entries exist - const store3 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - assert.equal(store3.values('dogs').length, 2); - assert.ok(store3.get('dogs', 'beagle')); - assert.ok(store3.get('dogs', 'poodle')); - }); - - it('clears the store on new build with force flag', async () => { - const tempDir = createTempDir(); - const dataStoreFile = new URL('./data-store.json', tempDir); + // Second build with force flag - should clear + const store2 = await mode.load(tempDir); - // First build - create data - const store1 = new MutableDataStore(); - store1.set('dogs', 'beagle', { - id: 'beagle', - data: { breed: 'Beagle' }, - }); - store1.metaStore().set('content-config-digest', 'digest1'); - - await fs.writeFile(dataStoreFile, store1.toString()); - - // Second build with force flag - should clear - const store2 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - - // Simulate force flag by clearing all - store2.clearAll(); - - // Add different data - store2.set('cats', 'siamese', { - id: 'siamese', - data: { breed: 'Siamese' }, - }); - - await fs.writeFile(dataStoreFile, store2.toString()); - - // Verify old data is gone, new data exists - const store3 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - assert.equal(store3.values('dogs').length, 0); - assert.equal(store3.values('cats').length, 1); - assert.ok(store3.get('cats', 'siamese')); - }); - - it('clears the store on new build if the content config has changed', async () => { - const tempDir = createTempDir(); - const dataStoreFile = new URL('./data-store.json', tempDir); - - // First build - const store1 = new MutableDataStore(); - store1.set('dogs', 'beagle', { - id: 'beagle', - data: { breed: 'Beagle' }, - }); - store1.metaStore().set('content-config-digest', 'digest1'); - - await fs.writeFile(dataStoreFile, store1.toString()); - - // Second build with different config digest - const store2 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - const previousDigest = store2.metaStore().get('content-config-digest'); - const newDigest = 'digest2'; - - if (previousDigest && previousDigest !== newDigest) { - // Content config changed, clear store + // Simulate force flag by clearing all store2.clearAll(); - } - - store2.metaStore().set('content-config-digest', newDigest); - // Add new data - store2.set('cats', 'tabby', { - id: 'tabby', - data: { breed: 'Tabby' }, - }); - - await fs.writeFile(dataStoreFile, store2.toString()); - - // Verify - const store3 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - assert.equal(store3.values('dogs').length, 0); // Old data cleared - assert.equal(store3.values('cats').length, 1); // New data exists - assert.equal(store3.metaStore().get('content-config-digest'), 'digest2'); - }); + // Add different data + store2.set('cats', 'siamese', { + id: 'siamese', + data: { breed: 'Siamese' }, + }); - it('clears the store on new build if the Astro config has changed', async () => { - const tempDir = createTempDir(); - const dataStoreFile = new URL('./data-store.json', tempDir); + await mode.save(store2, tempDir); - // First build - const store1 = new MutableDataStore(); - store1.set('dogs', 'beagle', { - id: 'beagle', - data: { breed: 'Beagle' }, + // Verify old data is gone, new data exists + const store3 = await mode.load(tempDir); + assert.equal(store3.values('dogs').length, 0); + assert.equal(store3.values('cats').length, 1); + assert.ok(store3.get('cats', 'siamese')); }); - store1.metaStore().set('astro-config-digest', 'astroDigest1'); - - await fs.writeFile(dataStoreFile, store1.toString()); - - // Second build with different astro config - const store2 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - const previousAstroDigest = store2.metaStore().get('astro-config-digest'); - const newAstroDigest = 'astroDigest2'; - if (previousAstroDigest && previousAstroDigest !== newAstroDigest) { - // Astro config changed, clear store - store2.clearAll(); - } - - store2.metaStore().set('astro-config-digest', newAstroDigest); - - // Add new data - store2.set('birds', 'robin', { - id: 'robin', - data: { name: 'Robin' }, + it('clears the store on new build if the content config has changed', async () => { + const tempDir = createTempDir(); + + // First build + let store1: MutableDataStore; + if (mode.label) { + store1 = await mode.load(tempDir); + } else { + store1 = new MutableDataStore(); + } + store1.set('dogs', 'beagle', { + id: 'beagle', + data: { breed: 'Beagle' }, + }); + store1.metaStore().set('content-config-digest', 'digest1'); + + await mode.save(store1, tempDir); + + // Second build with different config digest + const store2 = await mode.load(tempDir); + const previousDigest = store2.metaStore().get('content-config-digest'); + const newDigest = 'digest2'; + + if (previousDigest && previousDigest !== newDigest) { + // Content config changed, clear store + store2.clearAll(); + } + + store2.metaStore().set('content-config-digest', newDigest); + + // Add new data + store2.set('cats', 'tabby', { + id: 'tabby', + data: { breed: 'Tabby' }, + }); + + await mode.save(store2, tempDir); + + // Verify + const store3 = await mode.load(tempDir); + assert.equal(store3.values('dogs').length, 0); // Old data cleared + assert.equal(store3.values('cats').length, 1); // New data exists + assert.equal(store3.metaStore().get('content-config-digest'), 'digest2'); }); - await fs.writeFile(dataStoreFile, store2.toString()); - - // Verify - const store3 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - assert.equal(store3.values('dogs').length, 0); // Old data cleared - assert.equal(store3.values('birds').length, 1); // New data exists - assert.equal(store3.metaStore().get('astro-config-digest'), 'astroDigest2'); - }); - - it('can handle references being renamed after a build', async () => { - const tempDir = createTempDir(); - const dataStoreFile = new URL('./data-store.json', tempDir); - - // First build - entry with reference - const store1 = new MutableDataStore(); - store1.set('cats', 'siamese', { - id: 'siamese', - data: { breed: 'Siamese' }, - }); - store1.set('posts', 'post1', { - id: 'post1', - data: { - title: 'My Cat', - cat: { collection: 'cats', id: 'siamese' }, - }, + it('clears the store on new build if the Astro config has changed', async () => { + const tempDir = createTempDir(); + + // First build + let store1: MutableDataStore; + if (mode.label) { + store1 = await mode.load(tempDir); + } else { + store1 = new MutableDataStore(); + } + store1.set('dogs', 'beagle', { + id: 'beagle', + data: { breed: 'Beagle' }, + }); + store1.metaStore().set('astro-config-digest', 'astroDigest1'); + + await mode.save(store1, tempDir); + + // Second build with different astro config + const store2 = await mode.load(tempDir); + const previousAstroDigest = store2.metaStore().get('astro-config-digest'); + const newAstroDigest = 'astroDigest2'; + + if (previousAstroDigest && previousAstroDigest !== newAstroDigest) { + // Astro config changed, clear store + store2.clearAll(); + } + + store2.metaStore().set('astro-config-digest', newAstroDigest); + + // Add new data + store2.set('birds', 'robin', { + id: 'robin', + data: { name: 'Robin' }, + }); + + await mode.save(store2, tempDir); + + // Verify + const store3 = await mode.load(tempDir); + assert.equal(store3.values('dogs').length, 0); // Old data cleared + assert.equal(store3.values('birds').length, 1); // New data exists + assert.equal(store3.metaStore().get('astro-config-digest'), 'astroDigest2'); }); - await fs.writeFile(dataStoreFile, store1.toString()); - - // Second build - rename the cat entry - const store2 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - - // Remove old entry - store2.delete('cats', 'siamese'); - - // Add renamed entry - store2.set('cats', 'siamese-cat', { - id: 'siamese-cat', - data: { breed: 'Siamese' }, + it('can handle references being renamed after a build', async () => { + const tempDir = createTempDir(); + + // First build - entry with reference + let store1: MutableDataStore; + if (mode.label) { + store1 = await mode.load(tempDir); + } else { + store1 = new MutableDataStore(); + } + store1.set('cats', 'siamese', { + id: 'siamese', + data: { breed: 'Siamese' }, + }); + store1.set('posts', 'post1', { + id: 'post1', + data: { + title: 'My Cat', + cat: { collection: 'cats', id: 'siamese' }, + }, + }); + + await mode.save(store1, tempDir); + + // Second build - rename the cat entry + const store2 = await mode.load(tempDir); + + // Remove old entry + store2.delete('cats', 'siamese'); + + // Add renamed entry + store2.set('cats', 'siamese-cat', { + id: 'siamese-cat', + data: { breed: 'Siamese' }, + }); + + // Update the reference + const post = store2.get('posts', 'post1'); + if (post) { + post.data.cat = { collection: 'cats', id: 'siamese-cat' }; + store2.set('posts', 'post1', post); + } + + await mode.save(store2, tempDir); + + // Verify + const store3 = await mode.load(tempDir); + assert.ok(!store3.get('cats', 'siamese')); // Old entry gone + assert.ok(store3.get('cats', 'siamese-cat')); // New entry exists + + const updatedPost: any = store3.get('posts', 'post1'); + assert.equal(updatedPost.data.cat.id, 'siamese-cat'); // Reference updated }); - - // Update the reference - const post = store2.get('posts', 'post1'); - if (post) { - post.data.cat = { collection: 'cats', id: 'siamese-cat' }; - store2.set('posts', 'post1', post); - } - - await fs.writeFile(dataStoreFile, store2.toString()); - - // Verify - const store3 = await MutableDataStore.fromFile(fileURLToPath(dataStoreFile)); - assert.ok(!store3.get('cats', 'siamese')); // Old entry gone - assert.ok(store3.get('cats', 'siamese-cat')); // New entry exists - - const updatedPost: any = store3.get('posts', 'post1'); - assert.equal(updatedPost.data.cat.id, 'siamese-cat'); // Reference updated }); -}); +}