diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index aa1960a870..a055c142e7 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -90,10 +90,17 @@ defineConfig({ build: { outDir: "dist", // Output directory trailingSlash: false, // Add trailing slashes to URLs + serverExternalPackages: ["knex", "@prisma/client"], }, }); ``` +Use `serverExternalPackages` for npm packages that must run only on the server, +such as database, cache, or messaging clients. Veryfront leaves these imports +external so the runtime resolves the installed package instead of sending it +through the browser module CDN. Use package roots only. Do not include versions +or subpaths. + ### Layout ```ts diff --git a/src/config/README.md b/src/config/README.md index edb521fd44..98eed6b315 100644 --- a/src/config/README.md +++ b/src/config/README.md @@ -36,11 +36,11 @@ field. The complete validated config is also passed to extensions and included in render-cache identity, so compatibility-only fields cannot be removed as incidental cleanup. -| Ownership | Fields | -| ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Core runtime/build | `projectSlug`, `react.version`, `directories.app/pages/components`, `router`, `layout`, `app`, `experimental.esmLayouts/rsc`, `build.outDir`, `build.ssg`, `cache`, supported `dev` fields, `resolve.importMap`, `security`, `middleware.custom`, `fs.veryfront`, `fs.github`, AI primitive discovery, `client`, `styles.stylesheet`, `integrations`, `extensions`, and core `openapi` fields | -| CLI or diagnostics | `experimental.precompileMDX`, `generate.preferredRouter`, `ai.enabled`, and provider API-key checks | -| Accepted for extension compatibility, without built-in semantics | `title`, `description`, `directories.ai`, `theme.colors`, `build.trailingSlash/esbuild`, `dev.host/open/hmrPort`, `theming`, `assetPipeline`, tracing/metrics project config, `search`, `fs.local.baseDir`, `fs.memory`, provider defaults, `ai.work`, `ai.mcp`, and `openapi.mcp` | +| Ownership | Fields | +| ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Core runtime/build | `projectSlug`, `react.version`, `directories.app/pages/components`, `router`, `layout`, `app`, `experimental.esmLayouts/rsc`, `build.outDir`, `build.ssg`, `build.serverExternalPackages`, `cache`, supported `dev` fields, `resolve.importMap`, `security`, `middleware.custom`, `fs.veryfront`, `fs.github`, AI primitive discovery, `client`, `styles.stylesheet`, `integrations`, `extensions`, and core `openapi` fields | +| CLI or diagnostics | `experimental.precompileMDX`, `generate.preferredRouter`, `ai.enabled`, and provider API-key checks | +| Accepted for extension compatibility, without built-in semantics | `title`, `description`, `directories.ai`, `theme.colors`, `build.trailingSlash/esbuild`, `dev.host/open/hmrPort`, `theming`, `assetPipeline`, tracing/metrics project config, `search`, `fs.local.baseDir`, `fs.memory`, provider defaults, `ai.work`, `ai.mcp`, and `openapi.mcp` | Keep public documentation aligned with this table. Implementing a compatibility-only field requires an owned consumer and end-to-end tests; diff --git a/src/config/schemas/config.schema.test.ts b/src/config/schemas/config.schema.test.ts index 63ee9593fd..94090b9e2d 100644 --- a/src/config/schemas/config.schema.test.ts +++ b/src/config/schemas/config.schema.test.ts @@ -478,6 +478,33 @@ describe("configSchema", () => { ); }); + it("accepts bare package names in build.serverExternalPackages", () => { + const config = validateVeryfrontConfig({ + build: { + serverExternalPackages: ["knex", "@prisma/client"], + }, + }); + + assertEquals(config.build?.serverExternalPackages, ["knex", "@prisma/client"]); + }); + + it("rejects versions, subpaths, duplicates, and empty server external packages", () => { + for ( + const serverExternalPackages of [ + ["knex@3.1.0"], + ["@prisma/client/runtime"], + ["knex", "knex"], + [], + ] + ) { + assertThrows( + () => validateVeryfrontConfig({ build: { serverExternalPackages } }), + Error, + "Invalid veryfront.config at build.serverExternalPackages", + ); + } + }); + it("returns registered validation errors without retaining the full config", () => { const input = { dev: { port: "invalid" }, diff --git a/src/config/schemas/config.schema.ts b/src/config/schemas/config.schema.ts index 0ca2f1b5e4..46db19927e 100644 --- a/src/config/schemas/config.schema.ts +++ b/src/config/schemas/config.schema.ts @@ -52,6 +52,12 @@ import { } from "#veryfront/utils/discovery-path-policy.ts"; import { MAX_PATH_LENGTH_CHARS } from "#veryfront/utils/constants/limits.ts"; import { isCanonicalProjectRelativePath } from "#veryfront/utils/project-relative-path.ts"; +import { + hasUniqueServerExternalPackages, + isValidServerExternalPackageName, + MAX_SERVER_EXTERNAL_PACKAGE_COUNT, + MAX_SERVER_EXTERNAL_PACKAGE_NAME_LENGTH, +} from "#veryfront/config/server-external-packages.ts"; const integrationNames = new Set(ALL_INTEGRATION_NAMES); const MAX_CSRF_EXCLUDE_PATH_COUNT = 64; @@ -356,6 +362,25 @@ export const getVeryfrontConfigSchema = defineSchema((v) => .object({ outDir: v.string().optional(), trailingSlash: v.boolean().optional(), + /** Bare npm package roots that the runtime resolves instead of bundling. */ + serverExternalPackages: v + .array( + v + .string() + .min(1) + .max(MAX_SERVER_EXTERNAL_PACKAGE_NAME_LENGTH) + .refine( + isValidServerExternalPackageName, + "Expected a bare npm package name without a version or subpath", + ), + ) + .min(1) + .max(MAX_SERVER_EXTERNAL_PACKAGE_COUNT) + .refine( + hasUniqueServerExternalPackages, + "Server external package names must be unique", + ) + .optional(), /** * Generate static HTML for all routes during `veryfront build`. * Defaults to true; disabling it produces no pages, so only turn it diff --git a/src/config/server-external-packages.ts b/src/config/server-external-packages.ts new file mode 100644 index 0000000000..e41234e223 --- /dev/null +++ b/src/config/server-external-packages.ts @@ -0,0 +1,64 @@ +import { + primordialArrayPush, + primordialArraySort, +} from "#veryfront/platform/compat/primordials/array.ts"; + +export const MAX_SERVER_EXTERNAL_PACKAGE_COUNT = 128; +export const MAX_SERVER_EXTERNAL_PACKAGE_NAME_LENGTH = 214; + +const ObjectFreeze = Object.freeze; +const RegExpExec = RegExp.prototype.exec; +const ReflectApply = Reflect.apply; +const SetAdd = Set.prototype.add; +const SetHas = Set.prototype.has; +const SERVER_EXTERNAL_PACKAGE_PATTERN = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/; + +/** Return whether a value names one bare npm package root. */ +export function isValidServerExternalPackageName(value: string): boolean { + return value.length <= MAX_SERVER_EXTERNAL_PACKAGE_NAME_LENGTH && + ReflectApply(RegExpExec, SERVER_EXTERNAL_PACKAGE_PATTERN, [value]) !== null; +} + +/** Return whether every configured package name occurs exactly once. */ +export function hasUniqueServerExternalPackages(values: readonly string[]): boolean { + const seen = new Set(); + for (let index = 0; index < values.length; index++) { + const value = values[index]!; + if (ReflectApply(SetHas, seen, [value]) as boolean) return false; + ReflectApply(SetAdd, seen, [value]); + } + return true; +} + +/** Capture an immutable, order-independent package list for one transform graph. */ +export function canonicalizeServerExternalPackages( + values: readonly string[] | undefined, +): readonly string[] | undefined { + if (!values || values.length === 0) return undefined; + + const canonical: string[] = []; + const seen = new Set(); + for (let index = 0; index < values.length; index++) { + const value = values[index]!; + if (ReflectApply(SetHas, seen, [value]) as boolean) continue; + ReflectApply(SetAdd, seen, [value]); + primordialArrayPush(canonical, value); + } + primordialArraySort(canonical, (left, right) => left < right ? -1 : left > right ? 1 : 0); + return ObjectFreeze(canonical); +} + +/** Build a stable framed identity for cache keys whose output depends on this list. */ +export function buildServerExternalPackagesIdentity( + values: readonly string[] | undefined, +): string | undefined { + const canonical = canonicalizeServerExternalPackages(values); + if (!canonical) return undefined; + + let identity = ""; + for (let index = 0; index < canonical.length; index++) { + const value = canonical[index]!; + identity += `${value.length}:${value};`; + } + return identity; +} diff --git a/src/modules/react-loader/component-loader.ts b/src/modules/react-loader/component-loader.ts index a33ff05e70..b2e29cd988 100644 --- a/src/modules/react-loader/component-loader.ts +++ b/src/modules/react-loader/component-loader.ts @@ -53,6 +53,7 @@ export async function loadModuleFromSource( dev, contentSourceId: options?.contentSourceId, reactVersion: options?.reactVersion, + serverExternalPackages: options?.serverExternalPackages, moduleServerOrigin, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, @@ -71,6 +72,7 @@ export async function loadModuleFromSource( vendorBundleHash: options?.vendorBundleHash, ssr: false, reactVersion: options?.reactVersion, + serverExternalPackages: options?.serverExternalPackages, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, dependencyPinningSource, diff --git a/src/modules/react-loader/ssr-module-loader/loader.test.ts b/src/modules/react-loader/ssr-module-loader/loader.test.ts index fb38ec825b..94c1b01d3b 100644 --- a/src/modules/react-loader/ssr-module-loader/loader.test.ts +++ b/src/modules/react-loader/ssr-module-loader/loader.test.ts @@ -182,6 +182,14 @@ describe("SSRModuleLoader", { sanitizeResources: false, sanitizeOps: false }, () }), undefined, ); + const externalA = __ssrModuleLoaderInternals.getMdxEsmCacheVariant({ + serverExternalPackages: ["knex", "@prisma/client"], + }); + const externalB = __ssrModuleLoaderInternals.getMdxEsmCacheVariant({ + serverExternalPackages: ["@prisma/client", "knex"], + }); + assertEquals(externalB, externalA); + assert(externalA?.startsWith("on:server-externals-")); }); it("invalidates stale cache entries with missing local dependencies and retransforms", async () => { diff --git a/src/modules/react-loader/ssr-module-loader/loader.ts b/src/modules/react-loader/ssr-module-loader/loader.ts index a0aada73c4..238f1cf558 100644 --- a/src/modules/react-loader/ssr-module-loader/loader.ts +++ b/src/modules/react-loader/ssr-module-loader/loader.ts @@ -67,7 +67,7 @@ import { createDependencyHashCache, type DependencyHashCache, } from "#veryfront/cache/dependency-graph.ts"; -import { buildDependencyPinningCacheVariant } from "#veryfront/cache/keys/dependency-pinning.ts"; +import { getMdxModuleCacheVariant } from "#veryfront/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.ts"; const logger = rendererLogger.component("ssr-module-loader"); const CACHE_FILE_MISSING_PREFIX = "Cache file missing:"; @@ -136,11 +136,15 @@ function publishTransformCacheIfCurrent(input: { } function getMdxEsmCacheVariant( - options: Pick, + options: Pick< + SSRModuleLoaderOptions, + "dependencyPinningCacheKey" | "moduleServerOrigin" | "serverExternalPackages" + >, ): string | undefined { - return buildDependencyPinningCacheVariant( + return getMdxModuleCacheVariant( options.dependencyPinningCacheKey, options.moduleServerOrigin, + options.serverExternalPackages, ); } @@ -811,6 +815,7 @@ export class SSRModuleLoader { apiBaseUrl: this.options.apiBaseUrl, moduleServerOrigin: this.options.moduleServerOrigin, reactVersion: this.options.reactVersion, + serverExternalPackages: this.options.serverExternalPackages, dependencyHashCache, dependencyPinningCacheKey: this.options.dependencyPinningCacheKey, dependencyPinningDependencies: this.options.dependencyPinningDependencies, diff --git a/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.test.ts b/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.test.ts index 6c252f1334..646bfbf3a3 100644 --- a/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.test.ts +++ b/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.test.ts @@ -68,6 +68,43 @@ describe("SSRCacheManager", { sanitizeResources: false, sanitizeOps: false }, () } }); + it("separates SSR module cache identity by server external packages", async () => { + const projectDir = await makeTempDir({ prefix: "vf-ssr-cache-manager-" }); + const baseOptions = { + projectDir, + projectId: "project-a", + contentSourceId: "preview-main", + adapter: denoAdapter, + dev: true, + }; + + try { + const noExternals = new SSRCacheManager(baseOptions); + const externalReact = new SSRCacheManager({ + ...baseOptions, + serverExternalPackages: ["react"], + }); + const externalReactDom = new SSRCacheManager({ + ...baseOptions, + serverExternalPackages: ["react", "react-dom"], + }); + const reorderedExternals = new SSRCacheManager({ + ...baseOptions, + serverExternalPackages: ["react-dom", "react"], + }); + + assertNotEquals(noExternals.getConfigHash(), externalReact.getConfigHash()); + assertNotEquals(externalReact.getConfigHash(), externalReactDom.getConfigHash()); + assertEquals(externalReactDom.getConfigHash(), reorderedExternals.getConfigHash()); + assertNotEquals( + externalReact.getCacheKey("/project/pages/index.tsx"), + externalReactDom.getCacheKey("/project/pages/index.tsx"), + ); + } finally { + await remove(projectDir, { recursive: true }); + } + }); + it("recovers missing vfmod dependencies for redis cache entries", async () => { const projectDir = await makeTempDir({ prefix: "vf-ssr-cache-manager-" }); const distributedCache = new FakeDistributedCache(); diff --git a/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.ts b/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.ts index 6450c094c7..36084174ce 100644 --- a/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.ts +++ b/src/modules/react-loader/ssr-module-loader/ssr-cache-manager.ts @@ -10,7 +10,9 @@ import { RUNTIME_VERSION } from "#veryfront/utils/version.ts"; import { INVALID_ARGUMENT } from "#veryfront/errors"; import { buildSSRModuleCacheKey } from "#veryfront/cache/keys.ts"; +import { hashString } from "#veryfront/cache/hash.ts"; import { computeConfigHashSync } from "#veryfront/cache/config-hash.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; import { createFileSystem } from "#veryfront/platform/compat/fs.ts"; import { rendererLogger } from "#veryfront/utils"; import { hashCodeHex } from "#veryfront/utils/hash-utils.ts"; @@ -49,13 +51,19 @@ export class SSRCacheManager { /** Lazily compute config hash once per manager instance. */ getConfigHash(): string { if (!this.cachedConfigHash) { - this.cachedConfigHash = computeConfigHashSync({ + const baseConfigHash = computeConfigHashSync({ reactVersion: this.options.reactVersion, dev: this.options.dev, apiBaseUrl: this.options.apiBaseUrl, moduleServerOrigin: this.options.moduleServerOrigin, dependencyPinningCacheKey: this.options.dependencyPinningCacheKey, }); + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + this.options.serverExternalPackages, + ); + this.cachedConfigHash = serverExternalPackagesIdentity + ? `${baseConfigHash}:server-externals:${hashString(serverExternalPackagesIdentity)}` + : baseConfigHash; } return this.cachedConfigHash; } diff --git a/src/modules/react-loader/ssr-module-loader/types.ts b/src/modules/react-loader/ssr-module-loader/types.ts index 1c02309c09..796377f2e1 100644 --- a/src/modules/react-loader/ssr-module-loader/types.ts +++ b/src/modules/react-loader/ssr-module-loader/types.ts @@ -23,6 +23,8 @@ export interface SSRModuleLoaderOptions { contentSourceId?: string; /** React version for transforms (defaults to DEFAULT_REACT_VERSION) */ reactVersion?: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Stable VERYFRONT_DEPENDENCY_PINNING + package dependency-map state. */ dependencyPinningCacheKey?: string; /** Immutable package map paired with dependencyPinningCacheKey. */ diff --git a/src/modules/react-loader/types.ts b/src/modules/react-loader/types.ts index 7c6d6493b5..8bfdc95ad4 100644 --- a/src/modules/react-loader/types.ts +++ b/src/modules/react-loader/types.ts @@ -16,6 +16,8 @@ export interface LoadComponentOptions { contentSourceId?: string; /** React version for transforms (from project config) */ reactVersion?: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Internal stable flag + package dependency-map key for cache isolation. */ dependencyPinningCacheKey?: string; /** Immutable package map paired with dependencyPinningCacheKey. */ diff --git a/src/modules/server/module-batch-handler.test.ts b/src/modules/server/module-batch-handler.test.ts index 5f5d7b4a84..5ed3a46656 100644 --- a/src/modules/server/module-batch-handler.test.ts +++ b/src/modules/server/module-batch-handler.test.ts @@ -113,6 +113,26 @@ describe( buildModuleTransformCacheKey("project", "pages/index.js", false), ); }); + + it("isolates transforms by the configured server external package set", () => { + const args = [ + "project", + "pages/index.js", + false, + "off", + "release-a", + "content-a", + "https://app.example", + ] as const; + const baseline = buildBatchTransformCacheKey(...args); + const knex = buildBatchTransformCacheKey(...args, ["knex"]); + const prismaAndKnex = buildBatchTransformCacheKey(...args, ["@prisma/client", "knex"]); + const reordered = buildBatchTransformCacheKey(...args, ["knex", "@prisma/client"]); + + assertEquals(knex === baseline, false); + assertEquals(prismaAndKnex === knex, false); + assertEquals(reordered, prismaAndKnex); + }); }); describe("clearBatchCache / getBatchCacheStats", () => { diff --git a/src/modules/server/module-batch-handler.ts b/src/modules/server/module-batch-handler.ts index baa37819c7..050ff0e94a 100644 --- a/src/modules/server/module-batch-handler.ts +++ b/src/modules/server/module-batch-handler.ts @@ -55,6 +55,8 @@ import { resolveRequestedDependencyPinningSnapshot, } from "#veryfront/transforms/esm/package-registry.ts"; import type { VeryfrontConfig } from "#veryfront/config"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; +import { hashString } from "#veryfront/cache/hash.ts"; const logger = serverLogger.component("module-batch"); const DEPENDENCY_PIN_PATTERN = /^on:[A-Za-z0-9._-]+$/; @@ -101,17 +103,24 @@ export function buildBatchTransformCacheKey( contentSourceIdentity: string, sourceContentHash: string, moduleServerOrigin: string, + serverExternalPackages?: readonly string[], ): string { + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + serverExternalPackages, + ); + const scopedProjectKey = serverExternalPackagesIdentity + ? `${projectKey}:server-externals:${hashString(serverExternalPackagesIdentity)}` + : projectKey; const cacheVariant = buildDependencyPinningCacheVariant( dependencyPinningCacheKey, moduleServerOrigin, ); if (!cacheVariant) { - return buildModuleTransformCacheKey(projectKey, modulePath, isSSR); + return buildModuleTransformCacheKey(scopedProjectKey, modulePath, isSSR); } return buildModuleTransformCacheKey( - `${projectKey}:source:${ + `${scopedProjectKey}:source:${ encodeURIComponent(contentSourceIdentity) }:pins:${cacheVariant}:content:${sourceContentHash}`, modulePath, @@ -313,6 +322,7 @@ export function handleModuleBatch(req: Request, options: BatchHandlerOptions): P contentSourceIdentity, sourceContentHash, url.origin, + config?.build?.serverExternalPackages, ); if (canUseTransformCache) { const cachedCode = transformCache.get(cacheKey); @@ -344,6 +354,7 @@ export function handleModuleBatch(req: Request, options: BatchHandlerOptions): P dependencyPinningDependencies: dependencySnapshot.dependencies, dependencyPinningSource: dependencySource, moduleServerOrigin: url.origin, + serverExternalPackages: config?.build?.serverExternalPackages, }, ); const transformDurationMs = performance.now() - moduleStart; @@ -506,6 +517,7 @@ async function transformModule( dependencyPinningDependencies?: Readonly>; dependencyPinningSource?: DependencyPinningSourceInput; moduleServerOrigin: string; + serverExternalPackages?: readonly string[]; }, ): Promise { return transformModuleToServable({ @@ -526,6 +538,7 @@ async function transformModule( dependencyPinningCacheKey: options.dependencyPinningCacheKey, dependencyPinningDependencies: options.dependencyPinningDependencies, dependencyPinningSource: options.dependencyPinningSource, + serverExternalPackages: options.serverExternalPackages, }, isSSR: options.ssr, ssrRewriteOptions: options.ssr diff --git a/src/modules/server/module-response-cache.test.ts b/src/modules/server/module-response-cache.test.ts index 901966639e..ed5255cb2b 100644 --- a/src/modules/server/module-response-cache.test.ts +++ b/src/modules/server/module-response-cache.test.ts @@ -132,6 +132,26 @@ describe("release module response cache", () => { assertEquals(flagOff, unkeyed); }); + it("isolates responses by the configured server external package set", () => { + const baseline = buildReleaseModuleResponseCacheKey(baseKeyOptions("@vite/env")); + const knex = buildReleaseModuleResponseCacheKey({ + ...baseKeyOptions("@vite/env"), + serverExternalPackages: ["knex"], + }); + const prismaAndKnex = buildReleaseModuleResponseCacheKey({ + ...baseKeyOptions("@vite/env"), + serverExternalPackages: ["@prisma/client", "knex"], + }); + const reordered = buildReleaseModuleResponseCacheKey({ + ...baseKeyOptions("@vite/env"), + serverExternalPackages: ["knex", "@prisma/client"], + }); + + assertEquals(knex === baseline, false); + assertEquals(prismaAndKnex === knex, false); + assertEquals(reordered, prismaAndKnex); + }); + it("isolates pin-on responses by module server origin without changing flag-off keys", () => { const unkeyed = buildReleaseModuleResponseCacheKey( baseKeyOptions("@vite/env"), diff --git a/src/modules/server/module-response-cache.ts b/src/modules/server/module-response-cache.ts index 6424cf438c..3cba92da5e 100644 --- a/src/modules/server/module-response-cache.ts +++ b/src/modules/server/module-response-cache.ts @@ -5,6 +5,7 @@ import type { CacheBackend } from "#veryfront/cache/types.ts"; import { buildDependencyPinningCacheVariant } from "#veryfront/cache/keys/dependency-pinning.ts"; import { LRUCache } from "#veryfront/utils/lru-wrapper.ts"; import { TRANSFORM_DISTRIBUTED_TTL_SEC } from "#veryfront/utils/constants/cache.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; const RELEASE_MODULE_RESPONSE_CACHE_MAX_ENTRIES = 10_000; const RELEASE_MODULE_RESPONSE_CACHE_MAX_BYTES = 64 * 1024 * 1024; @@ -30,6 +31,7 @@ export interface ReleaseModuleResponseCacheKeyOptions { dependencyPinningCacheKey?: string; moduleServerOrigin?: string; releaseDependencyManifestVersion?: number | null; + serverExternalPackages?: readonly string[]; modulePath: string; } @@ -138,6 +140,9 @@ export function buildReleaseModuleResponseCacheKey( options.dependencyPinningCacheKey, options.moduleServerOrigin, ); + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + options.serverExternalPackages, + ); return [ "module-server-release-response", hashString(projectScope), @@ -145,6 +150,9 @@ export function buildReleaseModuleResponseCacheKey( options.runtimeVersion, options.reactVersion ?? "", ...(cacheVariant ? [`pins:${cacheVariant}`] : []), + ...(serverExternalPackagesIdentity + ? [`server-externals:${hashString(serverExternalPackagesIdentity)}`] + : []), options.releaseDependencyManifestVersion == null ? "" : `release-dependency-manifest:${options.releaseDependencyManifestVersion}`, diff --git a/src/modules/server/module-server-externals.test.ts b/src/modules/server/module-server-externals.test.ts new file mode 100644 index 0000000000..ebd57fa151 --- /dev/null +++ b/src/modules/server/module-server-externals.test.ts @@ -0,0 +1,37 @@ +import "#veryfront/schemas/_test-setup.ts"; +import { assertEquals, assertStringIncludes } from "#veryfront/testing/assert.ts"; +import { afterAll, describe, it } from "#veryfront/testing/bdd.ts"; +import { createMockAdapter } from "#veryfront/platform/adapters/mock.ts"; +import { serveModule } from "./module-server.ts"; + +describe("module server external packages", () => { + afterAll(async () => { + const { stop } = await import("veryfront/extensions/bundler"); + await stop(); + }); + + it("preserves configured server externals in served project modules", async () => { + const projectDir = "/server-external-package"; + const adapter = createMockAdapter(); + adapter.fs.files.set( + `${projectDir}/components/Database.ts`, + `import knex from "knex"; export default knex;\n`, + ); + + const response = await serveModule( + new Request("http://localhost:3000/_vf_modules/components/Database.js"), + { + projectId: "server-external-package", + projectDir, + adapter, + isLocalProject: true, + config: { build: { serverExternalPackages: ["knex"] } }, + }, + ); + + assertEquals(response.status, 200); + const text = await response.text(); + assertStringIncludes(text, 'from "knex"'); + assertEquals(text.includes("esm.sh/knex"), false); + }); +}); diff --git a/src/modules/server/module-server.ts b/src/modules/server/module-server.ts index 93ab90ce7b..fa6ca4bc46 100644 --- a/src/modules/server/module-server.ts +++ b/src/modules/server/module-server.ts @@ -591,6 +591,7 @@ export function serveModule(req: Request, options: ModuleServerOptions): Promise dependencyPinningCacheKey, dependencyPinningDependencies, dependencyPinningSource: dependencySource, + serverExternalPackages: config?.build?.serverExternalPackages, }, isSSR, ssrRewriteOptions: { @@ -722,6 +723,7 @@ export function serveModule(req: Request, options: ModuleServerOptions): Promise dependencyPinningCacheKey, dependencyPinningDependencies, dependencyPinningSource: dependencySource, + serverExternalPackages: config?.build?.serverExternalPackages, }, isSSR, ssrRewriteOptions: { @@ -848,6 +850,7 @@ export function serveModule(req: Request, options: ModuleServerOptions): Promise reactVersion: state.reactVersion, dependencyPinningCacheKey: state.dependencyPinningCacheKey, moduleServerOrigin: url.origin, + serverExternalPackages: config?.build?.serverExternalPackages, releaseDependencyManifestVersion, modulePath, }) @@ -1142,6 +1145,7 @@ export function serveModule(req: Request, options: ModuleServerOptions): Promise dependencyPinningCacheKey, dependencyPinningDependencies, dependencyPinningSource: dependencySource, + serverExternalPackages: config?.build?.serverExternalPackages, }; // The dev-module path has two post-steps that stay outside diff --git a/src/rendering/app-reserved.ts b/src/rendering/app-reserved.ts index be152235d8..8badd88642 100644 --- a/src/rendering/app-reserved.ts +++ b/src/rendering/app-reserved.ts @@ -92,6 +92,7 @@ export async function loadReservedWithPath( dependencyPinningDependencies?: Readonly>, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise<{ component: ReservedComponent; filePath: string } | null> { const join = (a: string, b: string) => `${a.replace(/\/$/, "")}/${b.replace(/^\//, "")}`; const candidateName = RESERVED_COMPONENTS[which]; @@ -109,6 +110,7 @@ export async function loadReservedWithPath( dev: true, contentSourceId, reactVersion, + serverExternalPackages, moduleServerOrigin, dependencyPinningCacheKey, dependencyPinningDependencies, @@ -139,6 +141,7 @@ export async function tryLoadReservedInDirs( dependencyPinningDependencies?: Readonly>, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise { const loaded = await loadReservedWithPath( dirs, @@ -153,6 +156,7 @@ export async function tryLoadReservedInDirs( dependencyPinningDependencies, dependencyPinningSource, moduleServerOrigin, + serverExternalPackages, ); return loaded?.component ?? null; } diff --git a/src/rendering/component-handling.test.ts b/src/rendering/component-handling.test.ts index 78b761e8d8..4814cde218 100644 --- a/src/rendering/component-handling.test.ts +++ b/src/rendering/component-handling.test.ts @@ -164,6 +164,44 @@ describe("rendering/component-handling", () => { ]); }); + it("caches client bundle transforms by the configured server external package set", async () => { + let transformCalls = 0; + const deps = { + transformToESM: ( + _source: string, + _filePath: string, + _projectDir: string, + _adapter: RuntimeAdapter, + options: { serverExternalPackages?: readonly string[] }, + ) => { + transformCalls++; + return Promise.resolve(options.serverExternalPackages?.join(",") ?? "baseline"); + }, + }; + const bundle = (serverExternalPackages?: readonly string[]) => + bundleComponentForClient( + "export default function Page() { return null; }", + "/project/app/server-external-identity.tsx", + "/project", + {} as RuntimeAdapter, + "https://modules.example.test", + "project-server-external-identity", + "19.1.0", + deps, + undefined, + "off", + undefined, + undefined, + serverExternalPackages, + ); + + assertEquals(await bundle(), "baseline"); + assertEquals(await bundle(["knex"]), "knex"); + assertEquals(await bundle(["knex", "@prisma/client"]), "knex,@prisma/client"); + assertEquals(await bundle(["@prisma/client", "knex"]), "knex,@prisma/client"); + assertEquals(transformCalls, 3); + }); + it("keeps mainline off identity and isolates enabled snapshots and origins", async () => { const transformed: string[] = []; const deps = { diff --git a/src/rendering/component-handling.ts b/src/rendering/component-handling.ts index ca098c1cfa..f64ac75ed9 100644 --- a/src/rendering/component-handling.ts +++ b/src/rendering/component-handling.ts @@ -15,6 +15,7 @@ import { getDependencyPinningCacheKey } from "#veryfront/transforms/esm/package- import type { DependencyPinningSourceInput } from "#veryfront/transforms/esm/package-registry.ts"; import { Singleflight } from "#veryfront/utils/singleflight.ts"; import { DEFAULT_REACT_VERSION } from "#veryfront/transforms/import-rewriter/url-builder.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; interface ComponentPageResult { pageElement: BundledReact.ReactElement; @@ -47,6 +48,7 @@ interface BundleComponentForClientDeps { moduleServerUrl?: string; moduleServerOrigin?: string; reactVersion?: string; + serverExternalPackages?: readonly string[]; dependencyPinningCacheKey?: string; dependencyPinningDependencies?: Readonly>; dependencyPinningSource?: DependencyPinningSourceInput; @@ -66,14 +68,23 @@ async function buildComponentHydrationCacheHash( source: string, moduleServerUrl?: string, reactVersion?: string, + serverExternalPackages?: readonly string[], ): Promise { const sourceHash = await computeHash(source); const effectiveReactVersion = reactVersion ?? DEFAULT_REACT_VERSION; - const cacheIdentity = JSON.stringify([ + const legacyCacheIdentity = [ sourceHash, moduleServerUrl ?? null, effectiveReactVersion, - ]); + ]; + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + serverExternalPackages, + ); + const cacheIdentity = JSON.stringify( + serverExternalPackagesIdentity + ? [...legacyCacheIdentity, serverExternalPackagesIdentity] + : legacyCacheIdentity, + ); return (await computeHash(cacheIdentity)).slice(0, 16); } @@ -108,6 +119,8 @@ export async function handleComponentPage( dependencyPinningDependencies?: Readonly>; /** Exact package source namespace paired with the immutable snapshot. */ dependencyPinningSource?: DependencyPinningSourceInput; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; }, ): Promise { try { @@ -134,6 +147,7 @@ export async function handleComponentPage( dependencyPinningCacheKey, options?.dependencyPinningDependencies, options?.dependencyPinningSource, + options?.serverExternalPackages, )); const { loadComponentFromSource } = await import("#veryfront/modules/react-loader/index.ts"); @@ -151,6 +165,7 @@ export async function handleComponentPage( ssr: true, contentSourceId: options?.contentSourceId, reactVersion: options?.reactVersion, + serverExternalPackages: options?.serverExternalPackages, dependencyPinningCacheKey, dependencyPinningDependencies: options?.dependencyPinningDependencies, dependencyPinningSource: options?.dependencyPinningSource, @@ -207,12 +222,14 @@ export async function bundleComponentForClient( dependencyPinningCacheKey = "off", dependencyPinningDependencies?: Readonly>, dependencyPinningSource?: DependencyPinningSourceInput, + serverExternalPackages?: readonly string[], ): Promise { try { const cacheHash = await buildComponentHydrationCacheHash( source, moduleServerUrl, reactVersion, + serverExternalPackages, ); const cacheKey = buildComponentCacheKey( projectId ?? projectDir, @@ -238,6 +255,7 @@ export async function bundleComponentForClient( moduleServerUrl, moduleServerOrigin, reactVersion, + serverExternalPackages, dependencyPinningCacheKey, dependencyPinningDependencies, dependencyPinningSource, diff --git a/src/rendering/layouts/layout-applicator.ts b/src/rendering/layouts/layout-applicator.ts index 21bd1d04b8..53726b8ba6 100644 --- a/src/rendering/layouts/layout-applicator.ts +++ b/src/rendering/layouts/layout-applicator.ts @@ -314,6 +314,7 @@ export class LayoutApplicator { this.dependencyPinningDependencies, this.dependencyPinningSource, this.requestUrl?.origin, + this.config, ); }, { @@ -438,6 +439,7 @@ export class LayoutApplicator { dependencyPinningCacheKey: this.dependencyPinningCacheKey, dependencyPinningDependencies: this.dependencyPinningDependencies, dependencyPinningSource: this.dependencyPinningSource, + serverExternalPackages: this.config?.build?.serverExternalPackages, }, ); } @@ -494,6 +496,7 @@ export class LayoutApplicator { dependencyPinningCacheKey: this.dependencyPinningCacheKey, dependencyPinningDependencies: this.dependencyPinningDependencies, dependencyPinningSource: this.dependencyPinningSource, + serverExternalPackages: this.config?.build?.serverExternalPackages, }, ); } catch (error) { diff --git a/src/rendering/layouts/utils/applicator.ts b/src/rendering/layouts/utils/applicator.ts index 6482e4a85d..860c592d7f 100644 --- a/src/rendering/layouts/utils/applicator.ts +++ b/src/rendering/layouts/utils/applicator.ts @@ -115,6 +115,7 @@ export function applyLayoutsESM( dependencyPinningDependencies, dependencyPinningSource, moduleServerOrigin, + config?.build?.serverExternalPackages, ), spanAttrs, ); @@ -184,6 +185,7 @@ export async function applyLayoutsFunctionBody( dependencyPinningDependencies?: Readonly>, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + config?: VeryfrontConfig, ): Promise { const React = await getProjectReact(reactVersion); let element = pageElement; @@ -232,6 +234,7 @@ export async function applyLayoutsFunctionBody( dependencyPinningDependencies, dependencyPinningSource, moduleServerOrigin, + config?.build?.serverExternalPackages, ); const child = ensureValidChild(element, React); diff --git a/src/rendering/layouts/utils/component-loader.test.ts b/src/rendering/layouts/utils/component-loader.test.ts index 973a176d9a..12b527356d 100644 --- a/src/rendering/layouts/utils/component-loader.test.ts +++ b/src/rendering/layouts/utils/component-loader.test.ts @@ -679,4 +679,47 @@ describe("rendering/layouts/utils/component-loader", () => { assertEquals(requestedKeys[1], requestedKeys[0]); assertEquals(requestedKeys[0]?.includes(":pins:"), false); }); + + it("isolates the TSX layout cache by the server external package set", async () => { + function CachedLayout() { + return null; + } + const requestedKeys: string[] = []; + const cache = { + get(key: string) { + requestedKeys.push(key); + return CachedLayout; + }, + set() {}, + delete() {}, + clear() {}, + }; + const adapter = { + fs: { + readFile: () => Promise.resolve("export default function Layout() { return null; }"), + }, + } as unknown as RuntimeAdapter; + const common = [ + "/project/layout.tsx", + "/project", + cache, + adapter, + "project-id", + "project-slug", + "preview-main", + "19.1.1", + undefined, + "off", + undefined, + undefined, + undefined, + ] as const; + + await loadTSXComponent(...common); + await loadTSXComponent(...common, ["knex", "@prisma/client"]); + await loadTSXComponent(...common, ["@prisma/client", "knex"]); + + assertEquals(requestedKeys[0] === requestedKeys[1], false); + assertEquals(requestedKeys[2], requestedKeys[1]); + }); }); diff --git a/src/rendering/layouts/utils/component-loader.ts b/src/rendering/layouts/utils/component-loader.ts index 418e93cb51..1356bc715d 100644 --- a/src/rendering/layouts/utils/component-loader.ts +++ b/src/rendering/layouts/utils/component-loader.ts @@ -26,6 +26,8 @@ import { } from "#veryfront/transforms/esm/package-registry.ts"; import { buildDependencyPinningCacheVariant } from "#veryfront/cache/keys/dependency-pinning.ts"; import { Singleflight } from "#veryfront/utils/singleflight.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; +import { hashString } from "#veryfront/cache/hash.ts"; const loadMdxLayoutLog = logger.component("load-mdx-layout"); const applyTsxLayoutLog = logger.component("apply-tsx-layout"); @@ -262,6 +264,7 @@ export async function loadTSXComponent( dependencyPinningDependencies?: Readonly>, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise { const source = await adapter.fs.readFile(componentPath); const dependencySnapshot = await resolveDependencyPinningSnapshot( @@ -277,7 +280,13 @@ export async function loadTSXComponent( dependencySnapshot.cacheKey, moduleServerOrigin, ); - const cacheKey = cacheVariant ? `${legacyCacheKey}:pins:${cacheVariant}` : legacyCacheKey; + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + serverExternalPackages, + ); + let cacheKey = cacheVariant ? `${legacyCacheKey}:pins:${cacheVariant}` : legacyCacheKey; + if (serverExternalPackagesIdentity) { + cacheKey += `:server-externals:${hashString(serverExternalPackagesIdentity)}`; + } const cached = cache.get(cacheKey); if (cached) return cached; @@ -300,6 +309,7 @@ export async function loadTSXComponent( ssr: true, contentSourceId, reactVersion, + serverExternalPackages, moduleServerOrigin, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, @@ -385,6 +395,7 @@ export function loadMDXLayout( dependencyPinningSource, moduleServerOrigin, isLocalProject, + serverExternalPackages: config?.build?.serverExternalPackages, })) as MDXModule; loadMdxLayoutLog.debug("loadModuleESM DONE", { @@ -451,6 +462,7 @@ export async function applyTSXLayout( dependencyPinningDependencies?: Readonly>, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise { const start = performance.now(); applyTsxLayoutLog.debug("START", { @@ -479,6 +491,7 @@ export async function applyTSXLayout( dependencyPinningDependencies, dependencyPinningSource, moduleServerOrigin, + serverExternalPackages, ); applyTsxLayoutLog.debug("loadTSXComponent DONE", { diff --git a/src/rendering/orchestrator/html.ts b/src/rendering/orchestrator/html.ts index 4795f44d0f..000c1a5005 100644 --- a/src/rendering/orchestrator/html.ts +++ b/src/rendering/orchestrator/html.ts @@ -671,6 +671,7 @@ export class HTMLGenerator { context.options?.dependencyPinningDependencies, context.options?.dependencyPinningSource, context.options?.url?.origin, + this.config.config?.build?.serverExternalPackages, ); if (!loaded) return null; diff --git a/src/rendering/orchestrator/layout.ts b/src/rendering/orchestrator/layout.ts index 063c6bd5eb..4f4dfcbc7b 100644 --- a/src/rendering/orchestrator/layout.ts +++ b/src/rendering/orchestrator/layout.ts @@ -220,6 +220,7 @@ export class LayoutOrchestrator { dependencyPinningDependencies, dependencyPinningSource, moduleServerOrigin, + this.config.config.build?.serverExternalPackages, ); return { type: "tsx" as const, path: componentPath, success: true }; } catch (error) { @@ -356,6 +357,7 @@ export class LayoutOrchestrator { dependencyPinningDependencies, dependencyPinningSource, requestUrl?.origin, + this.config.config.build?.serverExternalPackages, ), ) : this.config.componentRegistry; diff --git a/src/rendering/orchestrator/module-loader/index.ts b/src/rendering/orchestrator/module-loader/index.ts index 43a0530d4f..fecf60492e 100644 --- a/src/rendering/orchestrator/module-loader/index.ts +++ b/src/rendering/orchestrator/module-loader/index.ts @@ -136,6 +136,7 @@ export async function transformModuleWithDeps( mode, config.dependencyPinningCacheKey, config.moduleServerOrigin, + config.serverExternalPackages, ); const cachedPath = await resolveCachedModulePath({ @@ -148,6 +149,7 @@ export async function transformModuleWithDeps( reactVersion: config.reactVersion, dependencyPinningCacheKey: config.dependencyPinningCacheKey, moduleServerOrigin: config.moduleServerOrigin, + serverExternalPackages: config.serverExternalPackages, }); if (cachedPath) { // Replay the evidence this module produced when it was last resolved. A @@ -282,6 +284,7 @@ export async function transformModuleWithDeps( adapter, reactVersion: config.reactVersion, moduleServerOrigin: config.moduleServerOrigin, + serverExternalPackages: config.serverExternalPackages, dependencyPinningCacheKey: config.dependencyPinningCacheKey, dependencyPinningDependencies: config.dependencyPinningDependencies, dependencyPinningSource: config.dependencyPinningSource, @@ -302,6 +305,7 @@ export async function transformModuleWithDeps( reactVersion: config.reactVersion, moduleServerOrigin: config.moduleServerOrigin, dependencyPinningCacheKey: config.dependencyPinningCacheKey, + serverExternalPackages: config.serverExternalPackages, isCycleTarget: cycleTargets.has(filePath), unresolvedSpecifiers: [...moduleUnresolvedSpecifiers], }); @@ -323,6 +327,8 @@ export interface ModuleLoaderConfig { reactVersion?: string; /** Absolute request origin used to identify same-origin module URLs. */ moduleServerOrigin?: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Stable VERYFRONT_DEPENDENCY_PINNING + package dependency-map state. */ dependencyPinningCacheKey?: string; /** Immutable package map paired with dependencyPinningCacheKey. */ @@ -558,6 +564,7 @@ export async function loadModule( config.mode, config.dependencyPinningCacheKey, config.moduleServerOrigin, + config.serverExternalPackages, ), ); // tmpDir is the exact cache dir this module was registered under, so the @@ -571,6 +578,7 @@ export async function loadModule( buildModuleTransformCacheVariant( config.dependencyPinningCacheKey, config.moduleServerOrigin, + config.serverExternalPackages, ), ); diff --git a/src/rendering/orchestrator/module-loader/module-cache-lookup.test.ts b/src/rendering/orchestrator/module-loader/module-cache-lookup.test.ts index 07d9e251e4..a461f51876 100644 --- a/src/rendering/orchestrator/module-loader/module-cache-lookup.test.ts +++ b/src/rendering/orchestrator/module-loader/module-cache-lookup.test.ts @@ -2,7 +2,11 @@ import "#veryfront/schemas/_test-setup.ts"; import { assertEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import { join } from "#veryfront/compat/path/index.ts"; -import { getModuleCacheKey, resolveCachedModulePath } from "./module-cache-lookup.ts"; +import { + buildModuleTransformCacheVariant, + getModuleCacheKey, + resolveCachedModulePath, +} from "./module-cache-lookup.ts"; async function withCachedFile( content: string, @@ -86,6 +90,33 @@ describe("module-loader/module-cache-lookup", () => { assertEquals(flagOffWithOrigin, flagOff); }); + it("isolates module paths by the configured server external package set", () => { + const base = [ + "/project/app/page.tsx", + "project-id", + "/project", + "source-id", + "19.0.0", + "production", + "off", + undefined, + ] as const; + const baseline = getModuleCacheKey(...base); + const knex = getModuleCacheKey(...base, ["knex"]); + const combined = getModuleCacheKey(...base, ["knex", "@prisma/client"]); + const reordered = getModuleCacheKey(...base, ["@prisma/client", "knex"]); + + assertEquals(knex === baseline, false); + assertEquals(combined === knex, false); + assertEquals(reordered, combined); + assertEquals( + buildModuleTransformCacheVariant("off", undefined, ["knex"])?.startsWith( + "on:server-externals-", + ), + true, + ); + }); + it("returns a valid in-memory cached module path", async () => { await withCachedFile("export const ok = true;", async (cachedPath) => { const moduleCache = new Map([["cache-key", cachedPath]]); diff --git a/src/rendering/orchestrator/module-loader/module-cache-lookup.ts b/src/rendering/orchestrator/module-loader/module-cache-lookup.ts index 6efdeb246c..e46957492f 100644 --- a/src/rendering/orchestrator/module-loader/module-cache-lookup.ts +++ b/src/rendering/orchestrator/module-loader/module-cache-lookup.ts @@ -14,17 +14,19 @@ import { getMdxEsmCacheDir } from "#veryfront/utils/cache-dir.ts"; import { rendererLogger } from "#veryfront/utils"; import { REACT_DEFAULT_VERSION } from "#veryfront/utils/constants/cdn.ts"; import { UNRESOLVED_VF_MODULES_RE } from "./module-transform-cache.ts"; -import { buildDependencyPinningCacheVariant } from "#veryfront/cache/keys/dependency-pinning.ts"; +import { getMdxModuleCacheVariant } from "#veryfront/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.ts"; const logger = rendererLogger.component("module-loader"); export function buildModuleTransformCacheVariant( dependencyPinningCacheKey?: string, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): string | undefined { - return buildDependencyPinningCacheVariant( + return getMdxModuleCacheVariant( dependencyPinningCacheKey, moduleServerOrigin, + serverExternalPackages, ); } @@ -37,6 +39,7 @@ export function getModuleCacheKey( mode?: "development" | "production", dependencyPinningCacheKey?: string, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): string { const base = projectId ?? projectDir ?? "default"; const source = contentSourceId ?? "default"; @@ -49,6 +52,7 @@ export function getModuleCacheKey( const cacheVariant = buildModuleTransformCacheVariant( dependencyPinningCacheKey, moduleServerOrigin, + serverExternalPackages, ); return JSON.stringify(cacheVariant ? [...fields, cacheVariant, filePath] : [...fields, filePath]); } @@ -65,6 +69,7 @@ export interface ResolveCachedModulePathInput { reactVersion?: string; dependencyPinningCacheKey?: string; moduleServerOrigin?: string; + serverExternalPackages?: readonly string[]; moduleCache: Map; readTextFile?: (path: string) => Promise; fileSystem?: FileSystemReader; @@ -126,6 +131,7 @@ async function resolveMdxEsmCachedPath( buildModuleTransformCacheVariant( input.dependencyPinningCacheKey, input.moduleServerOrigin, + input.serverExternalPackages, ), ); diff --git a/src/rendering/orchestrator/module-loader/module-persistence.ts b/src/rendering/orchestrator/module-loader/module-persistence.ts index aaf3b8bf29..9e278f418c 100644 --- a/src/rendering/orchestrator/module-loader/module-persistence.ts +++ b/src/rendering/orchestrator/module-loader/module-persistence.ts @@ -77,6 +77,7 @@ export interface PersistTransformedModuleInput { reactVersion?: string; dependencyPinningCacheKey?: string; moduleServerOrigin?: string; + serverExternalPackages?: readonly string[]; /** Tenant-authored imports left unresolved in this module subtree. */ unresolvedSpecifiers?: readonly string[]; /** @@ -543,6 +544,7 @@ export async function persistTransformedModule( const cacheVariant = buildModuleTransformCacheVariant( input.dependencyPinningCacheKey, input.moduleServerOrigin, + input.serverExternalPackages, ); const outputRelativePath = cacheVariant ? join("_pins", encodeURIComponent(cacheVariant), relativePath) @@ -611,6 +613,7 @@ export async function persistTransformedModule( buildModuleTransformCacheVariant( input.dependencyPinningCacheKey, input.moduleServerOrigin, + input.serverExternalPackages, ), ); const cache = await getModulePathCache(input.tmpDir); diff --git a/src/rendering/orchestrator/module-loader/module-transform-cache.test.ts b/src/rendering/orchestrator/module-loader/module-transform-cache.test.ts index 8c29f540f2..9557c4b5e4 100644 --- a/src/rendering/orchestrator/module-loader/module-transform-cache.test.ts +++ b/src/rendering/orchestrator/module-loader/module-transform-cache.test.ts @@ -85,7 +85,7 @@ describe("module-loader/module-transform-cache", () => { assertEquals(observedSignal, transformController.signal); }); - it("isolates outer transform cache keys by React, runtime, and dependency-pin state", async () => { + it("isolates outer transform cache keys by React, runtime, pins, and server externals", async () => { const cacheKeys: string[] = []; const deps = createDeps({ getOrComputeTransform: async ( @@ -151,8 +151,20 @@ describe("module-loader/module-transform-cache", () => { dependencyPinningCacheKey: CHANGED_PIN_KEY, moduleServerOrigin: "https://b.example", }); + await transformModuleCodeWithCache({ + ...baseInput, + mode: "production", + reactVersion: "18.3.1", + serverExternalPackages: ["knex", "@prisma/client"], + }); + await transformModuleCodeWithCache({ + ...baseInput, + mode: "production", + reactVersion: "18.3.1", + serverExternalPackages: ["@prisma/client", "knex"], + }); - assertEquals(new Set(cacheKeys).size, 6); + assertEquals(new Set(cacheKeys).size, 7); }); it("preserves the legacy outer transform identity when pinning is off", async () => { diff --git a/src/rendering/orchestrator/module-loader/module-transform-cache.ts b/src/rendering/orchestrator/module-loader/module-transform-cache.ts index 8ec623f25c..0cd9c7f735 100644 --- a/src/rendering/orchestrator/module-loader/module-transform-cache.ts +++ b/src/rendering/orchestrator/module-loader/module-transform-cache.ts @@ -29,6 +29,7 @@ import { } from "#veryfront/transforms/import-rewriter/dependency-resolution.ts"; import { replaySSRDependencyResolutionObservations } from "#veryfront/transforms/import-rewriter/ssr-adapter.ts"; import { buildDependencyPinningCacheVariant } from "#veryfront/cache/keys/dependency-pinning.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; const logger = rendererLogger.component("module-loader"); @@ -61,6 +62,7 @@ interface TransformOptions { ssr: boolean; reactVersion?: string; moduleServerOrigin?: string; + serverExternalPackages?: readonly string[]; dependencyPinningCacheKey?: string; dependencyPinningDependencies?: Readonly>; dependencyPinningSource?: DependencyPinningSourceInput; @@ -143,6 +145,7 @@ export interface TransformModuleCodeWithCacheInput { adapter: RuntimeAdapter; reactVersion?: string; moduleServerOrigin?: string; + serverExternalPackages?: readonly string[]; dependencyPinningCacheKey?: string; dependencyPinningDependencies?: Readonly>; dependencyPinningSource?: DependencyPinningSourceInput; @@ -217,8 +220,14 @@ export async function transformModuleCodeWithCache( input.mode, reactVersion, ]; + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + input.serverExternalPackages, + ); + const extendedConfig = serverExternalPackagesIdentity + ? [...legacyConfig, `server-externals:${serverExternalPackagesIdentity}`] + : legacyConfig; const configHash = hashCodeHex(JSON.stringify( - cacheVariant ? [...legacyConfig, cacheVariant] : legacyConfig, + cacheVariant ? [...extendedConfig, cacheVariant] : extendedConfig, )); const cacheKey = generateTransformCacheKey( scopedPath, @@ -237,6 +246,7 @@ export async function transformModuleCodeWithCache( ssr: true, reactVersion, moduleServerOrigin, + serverExternalPackages: input.serverExternalPackages, dependencyPinningCacheKey: input.dependencyPinningCacheKey, dependencyPinningDependencies: input.dependencyPinningDependencies, dependencyPinningSource: input.dependencyPinningSource, diff --git a/src/rendering/orchestrator/pipeline.ts b/src/rendering/orchestrator/pipeline.ts index ab2fcf753d..547ada8765 100644 --- a/src/rendering/orchestrator/pipeline.ts +++ b/src/rendering/orchestrator/pipeline.ts @@ -314,6 +314,7 @@ export class RenderPipeline { moduleServerOrigin: dependencySnapshot.cacheKey.startsWith("on:") ? options?.url?.origin : undefined, + serverExternalPackages: this.config.config?.build?.serverExternalPackages, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, dependencyPinningSource, diff --git a/src/rendering/page-renderer.ts b/src/rendering/page-renderer.ts index 798c821510..51e025e616 100644 --- a/src/rendering/page-renderer.ts +++ b/src/rendering/page-renderer.ts @@ -97,6 +97,7 @@ export class PageRenderer { dependencyPinningDependencies, dependencyPinningSource, moduleServerOrigin, + this.config.build?.serverExternalPackages, ); return { ...createDefaultMDXComponents(), @@ -229,6 +230,7 @@ export class PageRenderer { dependencyPinningCacheKey: options?.dependencyPinningCacheKey, dependencyPinningDependencies: options?.dependencyPinningDependencies, dependencyPinningSource: options?.dependencyPinningSource, + serverExternalPackages: this.config.build?.serverExternalPackages, }, ), { "render.component_path": pageInfo.entity.path }, @@ -267,6 +269,7 @@ export class PageRenderer { projectSlug: options?.projectSlug, contentSourceId: options?.contentSourceId, reactVersion, + serverExternalPackages: this.config.build?.serverExternalPackages, dependencyPinningCacheKey: options?.dependencyPinningCacheKey, dependencyPinningDependencies: options?.dependencyPinningDependencies, dependencyPinningSource: options?.dependencyPinningSource, diff --git a/src/rendering/page-rendering.ts b/src/rendering/page-rendering.ts index 5d6ca5b731..294ea760a9 100644 --- a/src/rendering/page-rendering.ts +++ b/src/rendering/page-rendering.ts @@ -156,6 +156,8 @@ export function handleMDXPage( dependencyPinningDependencies?: Readonly>; /** Exact package source namespace paired with the immutable snapshot. */ dependencyPinningSource?: DependencyPinningSourceInput; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Server-trusted local-project identity for dev-only module-server fallback. */ isLocalProject?: boolean; }, @@ -179,6 +181,7 @@ export function handleMDXPage( projectSlug: options?.projectSlug, contentSourceId: options?.contentSourceId, reactVersion: options?.reactVersion, + serverExternalPackages: options?.serverExternalPackages, dependencyPinningCacheKey: options?.dependencyPinningCacheKey, dependencyPinningDependencies: options?.dependencyPinningDependencies, dependencyPinningSource: options?.dependencyPinningSource, diff --git a/src/rendering/ssr/component-registry.test.ts b/src/rendering/ssr/component-registry.test.ts index 228798ffb7..0618c112cd 100644 --- a/src/rendering/ssr/component-registry.test.ts +++ b/src/rendering/ssr/component-registry.test.ts @@ -434,6 +434,56 @@ describe("ComponentRegistry logic", () => { assertEquals(seenOrigins, ["https://a.example", "https://b.example"]); }); + it("isolates component maps by the server external package set", async () => { + const adapter = createMockAdapter(); + adapter.fs.files.set( + "/project/components/Button.tsx", + "export default function Button() { return null; }", + ); + const seenPackages: string[] = []; + const registry = new ComponentRegistry( + { registerModule: () => Promise.resolve() } as unknown as VirtualModuleSystem, + 3001, + adapter, + undefined, + undefined, + "project-id", + "branch:main", + (_source, _filePath, _projectDir, _adapter, options) => { + const packages = options?.serverExternalPackages?.join(",") ?? "baseline"; + seenPackages.push(packages); + const Component: React.ComponentType> = () => null; + Component.displayName = `Button(${packages})`; + return Promise.resolve(Component); + }, + ); + + await registry.loadFromDirectory("/project/components", true); + const baseline = await registry.prepareDependencySnapshot("off"); + const combined = await registry.prepareDependencySnapshot( + "off", + undefined, + undefined, + undefined, + ["knex", "@prisma/client"], + ); + const reordered = await registry.prepareDependencySnapshot( + "off", + undefined, + undefined, + undefined, + ["@prisma/client", "knex"], + ); + + assertEquals(combined === baseline, false); + assertEquals(reordered, combined); + assertEquals( + registry.getAllAsComponents(combined).Button?.displayName, + "Button(knex,@prisma/client)", + ); + assertEquals(seenPackages, ["baseline", "knex,@prisma/client"]); + }); + it("bounds retained dependency snapshots and evicts the least recently used map", async () => { const adapter = createMockAdapter(); adapter.fs.files.set( diff --git a/src/rendering/ssr/component-registry.ts b/src/rendering/ssr/component-registry.ts index 6b10a8e9e6..0e3c36844b 100644 --- a/src/rendering/ssr/component-registry.ts +++ b/src/rendering/ssr/component-registry.ts @@ -11,6 +11,8 @@ import { resolveDependencyPinningSnapshot, } from "#veryfront/transforms/esm/package-registry.ts"; import type { LoadComponentOptions } from "#veryfront/modules/react-loader/types.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; +import { hashString } from "#veryfront/cache/hash.ts"; interface DeferredComponentSource { source: string; @@ -208,15 +210,22 @@ export class ComponentRegistry { dependencyPinningDependencies?: Readonly>, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise { const dependencySnapshot = await resolveDependencyPinningSnapshot( (dependencyPinningSource ?? this.projectDir) || undefined, dependencyPinningCacheKey, dependencyPinningDependencies, ); - const snapshotKey = dependencySnapshot.cacheKey.startsWith("on:") && moduleServerOrigin + let snapshotKey = dependencySnapshot.cacheKey.startsWith("on:") && moduleServerOrigin ? `${dependencySnapshot.cacheKey}:origin:${encodeURIComponent(moduleServerOrigin)}` : dependencySnapshot.cacheKey; + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + serverExternalPackages, + ); + if (serverExternalPackagesIdentity) { + snapshotKey += `:server-externals:${hashString(serverExternalPackagesIdentity)}`; + } const sourceGeneration = this.componentSourceGeneration; if ( this.dependencySnapshotGenerations.get(snapshotKey) === sourceGeneration @@ -240,6 +249,7 @@ export class ComponentRegistry { sourceGeneration, dependencyPinningSource, moduleServerOrigin, + serverExternalPackages, ).finally(() => { if (this.dependencySnapshotLoads.get(loadKey) === load) { this.dependencySnapshotLoads.delete(loadKey); @@ -257,6 +267,7 @@ export class ComponentRegistry { sourceGeneration: number, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise { const adapter = this.adapter; if (!adapter) { @@ -282,6 +293,7 @@ export class ComponentRegistry { dependencySnapshot, dependencyPinningSource, moduleServerOrigin, + serverExternalPackages, ), ); @@ -390,6 +402,7 @@ export class ComponentRegistry { dependencySnapshot?: DependencyPinningSnapshot, dependencyPinningSource?: DependencyPinningSourceInput, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): LoadComponentOptions { return { projectId: this.projectId ?? projectRoot, @@ -401,6 +414,7 @@ export class ComponentRegistry { dependencyPinningCacheKey: dependencySnapshot?.cacheKey, dependencyPinningDependencies: dependencySnapshot?.dependencies, dependencyPinningSource, + serverExternalPackages, }; } diff --git a/src/server/handlers/request/project-run-execute.handler.ts b/src/server/handlers/request/project-run-execute.handler.ts index 16696d0c23..5e19a05e64 100644 --- a/src/server/handlers/request/project-run-execute.handler.ts +++ b/src/server/handlers/request/project-run-execute.handler.ts @@ -1145,6 +1145,7 @@ async function executeReleaseAssetBuildRun(input: { ssr: options.ssr, studioEmbed: false, reactVersion: options.reactVersion, + serverExternalPackages: releaseConfig.build?.serverExternalPackages, dependencyPinningCacheKey: options.dependencyPinningSnapshot?.cacheKey, dependencyPinningDependencies: options.dependencyPinningSnapshot?.dependencies, dependencyPinningSource: options.dependencyPinningSource, diff --git a/src/server/handlers/request/ssr/error-page-fallback.ts b/src/server/handlers/request/ssr/error-page-fallback.ts index a7802226cb..956ed3ec9b 100644 --- a/src/server/handlers/request/ssr/error-page-fallback.ts +++ b/src/server/handlers/request/ssr/error-page-fallback.ts @@ -321,6 +321,7 @@ async function loadErrorComponent( dev: isLocal, contentSourceId, reactVersion, + serverExternalPackages: ctx.config?.build?.serverExternalPackages, moduleServerOrigin, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, diff --git a/src/server/services/rsc/orchestrators/handler.test.ts b/src/server/services/rsc/orchestrators/handler.test.ts index 156302631c..c6cee0e23d 100644 --- a/src/server/services/rsc/orchestrators/handler.test.ts +++ b/src/server/services/rsc/orchestrators/handler.test.ts @@ -2,6 +2,7 @@ import "#veryfront/schemas/_test-setup.ts"; import "#veryfront/transforms/plugins/__tests__/code-parser-setup.ts"; import { afterAll, beforeEach, describe, it } from "#veryfront/testing/bdd.ts"; import { expect } from "#std/expect.ts"; +import { assertEquals } from "#veryfront/testing/assert.ts"; import { RSCDevServerHandler } from "./handler.ts"; import { DEPENDENCY_PINNING_ENV_FLAG } from "#veryfront/release-assets/constants.ts"; import { RSC_DEPENDENCY_PINNING_HEADER } from "#veryfront/rendering/rsc/constants.ts"; @@ -32,15 +33,22 @@ describe( expect(handler).toBeDefined(); }); - it("passes the trusted local-project identity to the render handler", () => { + it("passes project module options to the render handler", () => { const localHandler = new RSCDevServerHandler("/tmp/test-project", { isLocalProject: true, + config: { build: { serverExternalPackages: ["knex"] } }, }); - const isLocalProject = (localHandler as unknown as { - renderHandler: { moduleOptions: { isLocalProject?: boolean } }; - }).renderHandler.moduleOptions.isLocalProject; + const moduleOptions = (localHandler as unknown as { + renderHandler: { + moduleOptions: { + isLocalProject?: boolean; + serverExternalPackages?: readonly string[]; + }; + }; + }).renderHandler.moduleOptions; - expect(isLocalProject).toBe(true); + assertEquals(moduleOptions.isLocalProject, true); + assertEquals(moduleOptions.serverExternalPackages, ["knex"]); }); }); diff --git a/src/server/services/rsc/orchestrators/handler.ts b/src/server/services/rsc/orchestrators/handler.ts index 16fdda9dd9..f69baa7438 100644 --- a/src/server/services/rsc/orchestrators/handler.ts +++ b/src/server/services/rsc/orchestrators/handler.ts @@ -104,6 +104,7 @@ export class RSCDevServerHandler { projectId: options.projectId, projectSlug: options.projectSlug, contentSourceId: moduleContentSourceId, + serverExternalPackages: options.config?.build?.serverExternalPackages, dependencyPinningSource: this.dependencyPinningSource, isLocalProject, reactVersion: (snapshot) => this.getReactVersionForSnapshot(snapshot), diff --git a/src/server/services/rsc/orchestrators/render-handler.test.ts b/src/server/services/rsc/orchestrators/render-handler.test.ts index 2fc6aed11f..d5c87231c1 100644 --- a/src/server/services/rsc/orchestrators/render-handler.test.ts +++ b/src/server/services/rsc/orchestrators/render-handler.test.ts @@ -150,6 +150,7 @@ describe("server/services/rsc/orchestrators/render-handler", () => { `import value from "example-package"; export default function Page() { return value; }`, ); let observedVersion: string | undefined; + let observedServerExternalPackages: readonly string[] | undefined; const observedOrigins: Array = []; const Page = () => null; const renderer = { @@ -164,9 +165,11 @@ describe("server/services/rsc/orchestrators/render-handler", () => { runtimeAdapter: () => Promise.resolve(adapter), moduleLoader: (_source, _path, _projectDir, _adapter, options) => { observedVersion = options?.dependencyPinningDependencies?.["example-package"]; + observedServerExternalPackages = options?.serverExternalPackages; observedOrigins.push(options?.moduleServerOrigin); return Promise.resolve({ default: Page }); }, + serverExternalPackages: ["knex"], reactVersion: (snapshot) => Promise.resolve(snapshot.dependencies?.react ?? "19.1.1"), }, ); @@ -198,6 +201,7 @@ describe("server/services/rsc/orchestrators/render-handler", () => { RSC_DEPENDENCY_PINNING_HEADER, ); assertEquals(observedVersion, "1.0.0"); + assertEquals(observedServerExternalPackages, ["knex"]); assertEquals(observedOrigins, [ "https://preview-a.example", "https://preview-b.example", @@ -275,7 +279,7 @@ describe("server/services/rsc/orchestrators/render-handler", () => { } }); - it("threads trusted local-project identity into RSC MDX loading", async () => { + it("threads trusted project options into RSC MDX loading", async () => { const projectDir = await Deno.makeTempDir({ prefix: "vf-rsc-local-mdx-" }); await Deno.mkdir(`${projectDir}/app`, { recursive: true }); await Deno.writeTextFile(`${projectDir}/app/page.mdx`, "# Local RSC page"); @@ -285,9 +289,11 @@ describe("server/services/rsc/orchestrators/render-handler", () => { loadModuleESM: typeof mdxRenderer.loadModuleESM; }; let observedIsLocalProject: unknown; + let observedServerExternalPackages: readonly string[] | undefined; mutableRenderer.loadModuleESM = (_compiledProgramCode, options) => { const loadOptions = options as MDXLoadModuleOptions | undefined; observedIsLocalProject = loadOptions?.isLocalProject; + observedServerExternalPackages = loadOptions?.serverExternalPackages; return Promise.resolve({ default: () => null }); }; @@ -304,6 +310,7 @@ describe("server/services/rsc/orchestrators/render-handler", () => { projectSlug: "local-project", contentSourceId: "local-main", isLocalProject: true, + serverExternalPackages: ["knex"], }, ); await (handler as unknown as { @@ -316,6 +323,7 @@ describe("server/services/rsc/orchestrators/render-handler", () => { }).loadComponent("/", snapshot, "19.1.1", "http://localhost"); assertEquals(observedIsLocalProject, true); + assertEquals(observedServerExternalPackages, ["knex"]); } finally { mutableRenderer.loadModuleESM = originalLoadModuleESM; await Deno.remove(projectDir, { recursive: true }); diff --git a/src/server/services/rsc/orchestrators/render-handler.ts b/src/server/services/rsc/orchestrators/render-handler.ts index f5579f5df6..0c8f475973 100644 --- a/src/server/services/rsc/orchestrators/render-handler.ts +++ b/src/server/services/rsc/orchestrators/render-handler.ts @@ -32,6 +32,7 @@ export interface RenderHandlerModuleOptions { projectId?: string; projectSlug?: string; contentSourceId?: string; + serverExternalPackages?: readonly string[]; dependencyPinningSource?: DependencyPinningSourceInput; /** Server-trusted local-project identity for dev-only module-server fallback. */ isLocalProject?: boolean; @@ -162,6 +163,7 @@ export class RenderHandler { projectSlug: this.moduleOptions.projectSlug, contentSourceId: this.moduleOptions.contentSourceId, reactVersion, + serverExternalPackages: this.moduleOptions.serverExternalPackages, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, dependencyPinningSource: this.moduleOptions.dependencyPinningSource, @@ -188,6 +190,7 @@ export class RenderHandler { dev: this.mode === "development", mode: this.mode === "development" ? "preview" : "production", reactVersion, + serverExternalPackages: this.moduleOptions.serverExternalPackages, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, dependencyPinningSource: this.moduleOptions.dependencyPinningSource, diff --git a/src/transforms/esm/bundle-recovery.ts b/src/transforms/esm/bundle-recovery.ts index 57d1da5fc2..beda6d93d8 100644 --- a/src/transforms/esm/bundle-recovery.ts +++ b/src/transforms/esm/bundle-recovery.ts @@ -49,6 +49,7 @@ function createRecoveryCacheOptions( cacheDir, importMap: identity?.importMap ?? EMPTY_IMPORT_MAP, reactVersion: identity?.reactVersion, + serverExternalPackages: identity?.serverExternalPackages, }; } diff --git a/src/transforms/esm/http-cache-helpers.test.ts b/src/transforms/esm/http-cache-helpers.test.ts index 2875304e98..9ab921fd8c 100644 --- a/src/transforms/esm/http-cache-helpers.test.ts +++ b/src/transforms/esm/http-cache-helpers.test.ts @@ -96,6 +96,28 @@ describe("transforms/esm/http-cache-helpers", () => { ); }); + it("partitions HTTP modules by the configured server external package set", async () => { + const url = "https://modules.example.com/root.js"; + const importMap = { imports: {}, scopes: {} }; + const baseline = await buildHttpCacheIdentity(url, { importMap }); + const knex = await buildHttpCacheIdentity(url, { + importMap, + serverExternalPackages: ["knex"], + }); + const prismaAndKnex = await buildHttpCacheIdentity(url, { + importMap, + serverExternalPackages: ["@prisma/client", "knex"], + }); + const reordered = await buildHttpCacheIdentity(url, { + importMap, + serverExternalPackages: ["knex", "@prisma/client"], + }); + + assertNotEquals(knex, baseline); + assertNotEquals(prismaAndKnex, knex); + assertEquals(reordered, prismaAndKnex); + }); + it("does not consult mutable JSON or array hooks for final identities", async () => { const importMap = { imports: {}, scopes: {} }; const baseline = await buildHttpCacheIdentity( diff --git a/src/transforms/esm/http-cache-helpers.ts b/src/transforms/esm/http-cache-helpers.ts index be8919ed93..c2d7ed87fc 100644 --- a/src/transforms/esm/http-cache-helpers.ts +++ b/src/transforms/esm/http-cache-helpers.ts @@ -21,6 +21,7 @@ import { buildEsmShUrl } from "../import-rewriter/url-builder.ts"; import { parseBarePackageSpecifier } from "../shared/package-specifier.ts"; import { DEFAULT_REACT_VERSION, getReactImportMap } from "./react-cdn.ts"; import { computeHash } from "#veryfront/utils/hash-utils.ts"; +import { canonicalizeServerExternalPackages } from "#veryfront/config/server-external-packages.ts"; const logger = rendererLogger.component("http-cache"); const ArrayIncludes = Array.prototype.includes; @@ -253,6 +254,8 @@ export type CacheOptions = { importMap: ImportMapConfig; /** React version to use for esm.sh URLs (defaults to DEFAULT_REACT_VERSION) */ reactVersion?: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Absolute request origin used to identify same-origin module-server URLs. */ moduleServerOrigin?: string; /** Request-scoped dependency-pinning state used to isolate module-server URLs. */ @@ -263,7 +266,10 @@ export type CacheOptions = { onProgress?: TransformProgressListener; }; -export type HttpCacheIdentityOptions = Pick; +export type HttpCacheIdentityOptions = Pick< + CacheOptions, + "importMap" | "reactVersion" | "serverExternalPackages" +>; export interface HttpCacheIdentityMetadata extends HttpCacheIdentityOptions { url: string; @@ -296,6 +302,7 @@ function compareImportMapKeys(left: [string, string], right: [string, string]): const HTTP_IMPORT_MAP_FINGERPRINT_NAMESPACE = "veryfront:http-import-map:v2"; const HTTP_CACHE_IDENTITY_NAMESPACE = "veryfront:http-module:v2"; +const HTTP_CACHE_EXTERNAL_IDENTITY_NAMESPACE = "veryfront:http-module:v3"; const HTTP_CACHE_FILE_HASH_NAMESPACE = "veryfront:http-module-file:v2"; /** Build an order-independent fingerprint covering imports and scoped imports. */ @@ -368,7 +375,10 @@ function getHttpCacheRequestIdentityContext( * Callers must create a fresh prepared options object for each top-level request. */ export function prepareHttpCacheRequestOptions(options: T): T { - const prepared = { ...options } as T; + const prepared = { + ...options, + serverExternalPackages: canonicalizeServerExternalPackages(options.serverExternalPackages), + } as T; return attachHttpCacheRequestIdentityContext(prepared, {}); } @@ -415,9 +425,20 @@ export async function buildHttpCacheIdentity( const normalizedUrl = normalizeHttpUrl(effective.url); const importMapFingerprint = await getRequestImportMapFingerprint(url, effective.options); const reactVersion = effective.options.reactVersion; - return `${HTTP_CACHE_IDENTITY_NAMESPACE}:[${JSONStringify(normalizedUrl)},${ + const base = `[${JSONStringify(normalizedUrl)},${ reactVersion === undefined ? "null" : JSONStringify(reactVersion) - },${JSONStringify(importMapFingerprint)}]`; + },${JSONStringify(importMapFingerprint)}`; + const serverExternalPackages = canonicalizeServerExternalPackages( + effective.options.serverExternalPackages, + ); + if (!serverExternalPackages) return `${HTTP_CACHE_IDENTITY_NAMESPACE}:${base}]`; + + let identity = `${HTTP_CACHE_EXTERNAL_IDENTITY_NAMESPACE}:${base},[`; + for (let index = 0; index < serverExternalPackages.length; index++) { + if (index > 0) identity += ","; + identity += JSONStringify(serverExternalPackages[index]); + } + return `${identity}]]`; } /** Build recoverable metadata while reusing the request graph's import-map fingerprint. */ @@ -430,6 +451,9 @@ export async function buildHttpCacheIdentityMetadata( url: normalizeHttpUrl(effective.url), importMap: effective.options.importMap, reactVersion: effective.options.reactVersion, + serverExternalPackages: canonicalizeServerExternalPackages( + effective.options.serverExternalPackages, + ), importMapFingerprint: await getRequestImportMapFingerprint(url, effective.options), }; } diff --git a/src/transforms/esm/http-cache-wrapper.test.ts b/src/transforms/esm/http-cache-wrapper.test.ts index f0b2cd5140..c14e7c9db8 100644 --- a/src/transforms/esm/http-cache-wrapper.test.ts +++ b/src/transforms/esm/http-cache-wrapper.test.ts @@ -146,6 +146,7 @@ describe("transforms/esm/http-cache-wrapper", () => { url, importMap, importMapFingerprint, + serverExternalPackages: ["knex"], }); } @@ -158,6 +159,12 @@ describe("transforms/esm/http-cache-wrapper", () => { identityValues.every((value) => value.importMapFingerprint === importMapFingerprint), true, ); + assertEquals( + identityValues.every((value) => + JSON.stringify(value.serverExternalPackages) === JSON.stringify(["knex"]) + ), + true, + ); const importMapEntries = [...backend.entries] .filter(([key]) => key.includes(":import-map:")); @@ -166,6 +173,7 @@ describe("transforms/esm/http-cache-wrapper", () => { assertEquals(await httpBundleCache.getIdentityMetadata("bundle-a"), { url: "https://modules.example.com/a.js", reactVersion: undefined, + serverExternalPackages: ["knex"], importMap, importMapFingerprint, }); diff --git a/src/transforms/esm/http-cache-wrapper.ts b/src/transforms/esm/http-cache-wrapper.ts index a48a157f3d..e18335b799 100644 --- a/src/transforms/esm/http-cache-wrapper.ts +++ b/src/transforms/esm/http-cache-wrapper.ts @@ -37,6 +37,12 @@ import { import { looksLikeHtmlContent as looksLikeHtml } from "./html-content.ts"; import { fingerprintImportMap, type HttpCacheIdentityMetadata } from "./http-cache-helpers.ts"; import type { ImportMapConfig } from "#veryfront/modules/import-map/types.ts"; +import { + canonicalizeServerExternalPackages, + hasUniqueServerExternalPackages, + isValidServerExternalPackageName, + MAX_SERVER_EXTERNAL_PACKAGE_COUNT, +} from "#veryfront/config/server-external-packages.ts"; const logger = rendererLogger.component("http-cache-wrapper"); @@ -89,9 +95,25 @@ function parseImportMap(value: unknown): ImportMapConfig | null { interface HttpCacheIdentityReference { url: string; reactVersion?: string; + serverExternalPackages?: readonly string[]; importMapFingerprint: string; } +function parseServerExternalPackages(value: unknown): readonly string[] | null | undefined { + if (value === undefined) return undefined; + if ( + !Array.isArray(value) || value.length === 0 || + value.length > MAX_SERVER_EXTERNAL_PACKAGE_COUNT + ) return null; + const packages = value as unknown[]; + for (let index = 0; index < packages.length; index++) { + const entry = packages[index]; + if (typeof entry !== "string" || !isValidServerExternalPackageName(entry)) return null; + } + if (!hasUniqueServerExternalPackages(packages as string[])) return null; + return canonicalizeServerExternalPackages(packages as string[]); +} + function parseIdentityMetadata( value: string, ): HttpCacheIdentityMetadata | HttpCacheIdentityReference | null { @@ -99,6 +121,8 @@ function parseIdentityMetadata( const parsed = JSON.parse(value) as Record; if (!parsed || typeof parsed !== "object" || typeof parsed.url !== "string") return null; if (parsed.reactVersion !== undefined && typeof parsed.reactVersion !== "string") return null; + const serverExternalPackages = parseServerExternalPackages(parsed.serverExternalPackages); + if (serverExternalPackages === null) return null; // Backward compatibility for v2 records written before import maps were shared. if (parsed.importMap !== undefined) { @@ -107,6 +131,7 @@ function parseIdentityMetadata( ? { url: parsed.url, reactVersion: parsed.reactVersion as string | undefined, + ...(serverExternalPackages ? { serverExternalPackages } : {}), importMap, } : null; @@ -116,6 +141,7 @@ function parseIdentityMetadata( return { url: parsed.url, reactVersion: parsed.reactVersion as string | undefined, + ...(serverExternalPackages ? { serverExternalPackages } : {}), importMapFingerprint: parsed.importMapFingerprint, }; } catch { @@ -393,6 +419,7 @@ class HttpBundleCache { JSON.stringify({ url: identityMetadata.url, reactVersion: identityMetadata.reactVersion, + serverExternalPackages: identityMetadata.serverExternalPackages, importMapFingerprint, }), ttl, diff --git a/src/transforms/esm/specifier-resolver.test.ts b/src/transforms/esm/specifier-resolver.test.ts index 9cc0a8867e..caeaf8cf30 100644 --- a/src/transforms/esm/specifier-resolver.test.ts +++ b/src/transforms/esm/specifier-resolver.test.ts @@ -592,6 +592,36 @@ describe("transforms/esm/specifier-resolver", () => { } }); + it("leaves configured server external packages out of the HTTP cache graph", async () => { + const cases = [ + ["knex", 'export const load = () => import("knex");'], + ["npm:knex@3.1.0", 'export const load = () => import("npm:knex@3.1.0");'], + [ + "@prisma/client/runtime/library", + 'export const load = () => import("@prisma/client/runtime/library");', + ], + ] as const; + + for (const [specifier, code] of cases) { + let cacheCalls = 0; + const result = await buildReplacements( + code, + "https://esm.sh/parent@1/index.js", + { + ...defaultOptions, + serverExternalPackages: ["knex", "@prisma/client"], + }, + async () => { + cacheCalls++; + return null; + }, + ); + + assertEquals(cacheCalls, 0, `${specifier} must not hit esm.sh`); + assertEquals(result.replacements.size, 0, `${specifier} must be left in place`); + } + }); + it("aborts when a dynamic bare specifier fails to resolve", async () => { const code = `export const load = () => import("some-package");`; await assertRejects( diff --git a/src/transforms/esm/specifier-resolver.ts b/src/transforms/esm/specifier-resolver.ts index 787a68d9eb..dd6419f532 100644 --- a/src/transforms/esm/specifier-resolver.ts +++ b/src/transforms/esm/specifier-resolver.ts @@ -187,7 +187,10 @@ async function resolveSpecifier( ? stringSlice(specifier, 4) : specifier; const serverOnlyParsed = parseBarePackageSpecifier(serverOnlyCandidate); - if (serverOnlyParsed && isServerOnlyPackage(serverOnlyParsed.packageName)) return null; + if ( + serverOnlyParsed && + isServerOnlyPackage(serverOnlyParsed.packageName, options.serverExternalPackages) + ) return null; if (isInternalBare(specifier)) { const mapped = resolveImport(specifier, options.importMap); diff --git a/src/transforms/esm/types.ts b/src/transforms/esm/types.ts index a9085e750c..6d370ebe1e 100644 --- a/src/transforms/esm/types.ts +++ b/src/transforms/esm/types.ts @@ -21,6 +21,8 @@ export interface TransformOptions { studioEmbed?: boolean; /** React version for transforms (from project config, defaults to DEFAULT_REACT_VERSION) */ reactVersion?: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Immutable import-map snapshot already selected for this render. */ preloadedImportMap?: ImportMapConfig; /** Adapter used to load and cache the project import map before SSR cache identity. */ diff --git a/src/transforms/import-rewriter/strategies/bare-strategy.test.ts b/src/transforms/import-rewriter/strategies/bare-strategy.test.ts index ca0037b8df..24dc972cac 100644 --- a/src/transforms/import-rewriter/strategies/bare-strategy.test.ts +++ b/src/transforms/import-rewriter/strategies/bare-strategy.test.ts @@ -141,6 +141,32 @@ describe("BareStrategy", () => { assertEquals(result.specifier, null); }); + it("leaves a configured server external package and its subpaths external", () => { + const ctx = makeCtx({ + target: "browser", + serverExternalPackages: ["knex", "@prisma/client"], + }); + + assertEquals(bareStrategy.rewrite(makeInfo("knex"), ctx).specifier, null); + assertEquals(bareStrategy.rewrite(makeInfo("npm:knex@3.1.0"), ctx).specifier, null); + assertEquals( + bareStrategy.rewrite(makeInfo("@prisma/client/runtime/library"), ctx).specifier, + null, + ); + }); + + it("still rewrites an undeclared package when server externals are configured", () => { + const result = bareStrategy.rewrite( + makeInfo("sequelize"), + makeCtx({ target: "browser", serverExternalPackages: ["knex"] }), + ); + + assertEquals( + result.specifier, + "https://esm.sh/sequelize?external=react,react-dom&target=es2022", + ); + }); + // The `npm:` scheme alone does not mean server-only. A browser-safe package // imported Deno-style (`npm:zod@4.0.0`) must still flow through esm.sh — the // `npm:` prefix is stripped and the package rewritten like a bare import, so diff --git a/src/transforms/import-rewriter/strategies/bare-strategy.ts b/src/transforms/import-rewriter/strategies/bare-strategy.ts index 027c679308..a84ea279b9 100644 --- a/src/transforms/import-rewriter/strategies/bare-strategy.ts +++ b/src/transforms/import-rewriter/strategies/bare-strategy.ts @@ -120,7 +120,7 @@ export class BareStrategy implements ImportRewriteStrategy { // resolves them natively (node_modules on Node, npm: on Deno). The // framework's adapters only `import()` these behind a lazy, configured code // path, so an app that does not use the backend never loads them at all. - if (parsed && isServerOnlyPackage(parsed.packageName)) { + if (parsed && isServerOnlyPackage(parsed.packageName, ctx.serverExternalPackages)) { return { specifier: null }; } diff --git a/src/transforms/import-rewriter/types.ts b/src/transforms/import-rewriter/types.ts index 7631f813ff..5c9ae3e513 100644 --- a/src/transforms/import-rewriter/types.ts +++ b/src/transforms/import-rewriter/types.ts @@ -38,6 +38,8 @@ export interface RewriteContext { apiBaseUrl?: string; /** React version to use for esm.sh URLs */ reactVersion: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Stable dependency-pinning key paired with the immutable dependency map. */ dependencyPinningCacheKey?: string; /** Immutable dependency map captured with dependencyPinningCacheKey. */ diff --git a/src/transforms/mdx/esm-module-loader/import-transformer.test.ts b/src/transforms/mdx/esm-module-loader/import-transformer.test.ts index 929136133a..6a4cf4c636 100644 --- a/src/transforms/mdx/esm-module-loader/import-transformer.test.ts +++ b/src/transforms/mdx/esm-module-loader/import-transformer.test.ts @@ -180,6 +180,25 @@ describe("rewriteMdxRootDependencyImports", () => { ); }); + it("keeps configured server external packages unchanged", async () => { + const code = [ + `import knex from "knex";`, + `import prisma from "@prisma/client";`, + `const queryBuilder = import("knex/query");`, + ].join("\n"); + + const result = await rewriteMdxRootDependencyImports( + code, + { imports: {} }, + { + ...baseOptions, + serverExternalPackages: ["knex", "@prisma/client"], + }, + ); + + assertEquals(result, code); + }); + it("keeps every supported bare Node builtin and its node: form unchanged", async () => { const code = NODE_BUILTINS.flatMap((specifier) => [ `import "${specifier}";`, diff --git a/src/transforms/mdx/esm-module-loader/import-transformer.ts b/src/transforms/mdx/esm-module-loader/import-transformer.ts index 2a9576c1ef..c7c261c213 100644 --- a/src/transforms/mdx/esm-module-loader/import-transformer.ts +++ b/src/transforms/mdx/esm-module-loader/import-transformer.ts @@ -125,6 +125,7 @@ export interface MdxRootDependencyRewriteOptions { dependencyPinningCacheKey?: string; dependencyPinningDependencies?: Readonly>; dependencyPinningSource?: DependencyPinningSourceInput; + serverExternalPackages?: readonly string[]; onDependencyResolutionObserved?: ( observation: DependencyResolutionObservation, ) => void; @@ -153,6 +154,7 @@ export async function rewriteMdxRootDependencyImports( dependencyPinningCacheKey: options.dependencyPinningCacheKey, dependencyPinningDependencies: options.dependencyPinningDependencies, dependencyPinningSource: options.dependencyPinningSource, + serverExternalPackages: options.serverExternalPackages, onDependencyResolutionObserved: options.onDependencyResolutionObserved, }); } diff --git a/src/transforms/mdx/esm-module-loader/loader-helpers.ts b/src/transforms/mdx/esm-module-loader/loader-helpers.ts index ca42f65bb8..9d03eb6a85 100644 --- a/src/transforms/mdx/esm-module-loader/loader-helpers.ts +++ b/src/transforms/mdx/esm-module-loader/loader-helpers.ts @@ -197,6 +197,7 @@ export async function processVfModuleImports( contentSourceId: context.contentSourceId, isLocalProject: context.isLocalProject, reactVersion: context.reactVersion, + serverExternalPackages: context.serverExternalPackages, moduleServerOrigin: context.moduleServerOrigin, dependencyPinningCacheKey: context.dependencyPinningCacheKey, dependencyPinningDependencies: context.dependencyPinningDependencies, diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.test.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.test.ts index a8a4d4307d..65dfa2eec9 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.test.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.test.ts @@ -140,6 +140,26 @@ describe("transforms/mdx/esm-module-loader/module-fetcher/cache-keys", () => { assertEquals(originA === originB, false); assertEquals(flagOffWithOrigin, flagOff); }); + + it("isolates transforms by the configured server external package set", () => { + const base = [ + "proj", + "preview-main", + "19.1.1", + "lib/utils.ts", + "abc123", + "off", + undefined, + ] as const; + const baseline = getTransformCacheKey(...base); + const knex = getTransformCacheKey(...base, ["knex"]); + const combined = getTransformCacheKey(...base, ["knex", "@prisma/client"]); + const reordered = getTransformCacheKey(...base, ["@prisma/client", "knex"]); + + assertEquals(knex === baseline, false); + assertEquals(combined === knex, false); + assertEquals(reordered, combined); + }); }); describe("getVersionedPathCacheKey", () => { @@ -206,5 +226,34 @@ describe("transforms/mdx/esm-module-loader/module-fetcher/cache-keys", () => { assertEquals(originA === originB, false); assertEquals(flagOffWithOrigin, flagOff); }); + + it("isolates local paths by the configured server external package set", () => { + const baseline = getVersionedPathCacheKey("lib/utils.ts", "19.1.1"); + const knex = getVersionedPathCacheKey( + "lib/utils.ts", + "19.1.1", + "off", + undefined, + ["knex"], + ); + const combined = getVersionedPathCacheKey( + "lib/utils.ts", + "19.1.1", + "off", + undefined, + ["knex", "@prisma/client"], + ); + const reordered = getVersionedPathCacheKey( + "lib/utils.ts", + "19.1.1", + "off", + undefined, + ["@prisma/client", "knex"], + ); + + assertEquals(knex === baseline, false); + assertEquals(combined === knex, false); + assertEquals(reordered, combined); + }); }); }); diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.ts index 67ecb9f564..65329102c1 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/cache-keys.ts @@ -6,15 +6,24 @@ import { buildMdxEsmPathCacheKey, buildMdxEsmTransformCacheKey } from "../cache-format.ts"; import { buildDependencyPinningCacheVariant } from "#veryfront/cache/keys/dependency-pinning.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; +import { hashString } from "#veryfront/cache/hash.ts"; export function getMdxModuleCacheVariant( dependencyPinningCacheKey?: string, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): string | undefined { - return buildDependencyPinningCacheVariant( + const pinVariant = buildDependencyPinningCacheVariant( dependencyPinningCacheKey, moduleServerOrigin, ); + const externalIdentity = buildServerExternalPackagesIdentity(serverExternalPackages); + if (!externalIdentity) return pinVariant; + + // MDX cache variants use the `on:` prefix to opt into the variant segment. + const externalVariant = `on:server-externals-${hashString(externalIdentity)}`; + return pinVariant ? `${pinVariant}:${externalVariant}` : externalVariant; } /** @@ -32,6 +41,7 @@ export function getTransformCacheKey( contentHash: string, dependencyPinningCacheKey?: string, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): string { return buildMdxEsmTransformCacheKey( projectId, @@ -39,7 +49,11 @@ export function getTransformCacheKey( reactVersion, normalizedPath, contentHash, - getMdxModuleCacheVariant(dependencyPinningCacheKey, moduleServerOrigin), + getMdxModuleCacheVariant( + dependencyPinningCacheKey, + moduleServerOrigin, + serverExternalPackages, + ), ); } @@ -48,10 +62,15 @@ export function getVersionedPathCacheKey( reactVersion: string, dependencyPinningCacheKey?: string, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): string { return buildMdxEsmPathCacheKey( normalizedPath, reactVersion, - getMdxModuleCacheVariant(dependencyPinningCacheKey, moduleServerOrigin), + getMdxModuleCacheVariant( + dependencyPinningCacheKey, + moduleServerOrigin, + serverExternalPackages, + ), ); } diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/distributed-cache.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/distributed-cache.ts index f23656fed4..897dfbff2e 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/distributed-cache.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/distributed-cache.ts @@ -116,6 +116,7 @@ export async function readDistributedCache( projectDir: string, reactVersion: string | undefined, log: Logger, + serverExternalPackages?: readonly string[], ): Promise { const distributedCache = await getDistributedTransformBackend(); if (!distributedCache) return null; @@ -225,6 +226,7 @@ export async function readDistributedCache( cacheDir: getHttpBundleCacheDir(), importMap, reactVersion, + serverExternalPackages, }); if (cacheResult.code !== moduleCode) { log.debug(`${LOG_PREFIX_MDX_LOADER} Converted HTTP imports from legacy cache entry`, { diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/http-fallback.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/http-fallback.ts index c437b2f1fb..f081f38bf2 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/http-fallback.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/http-fallback.ts @@ -27,6 +27,7 @@ export interface ResolveUnresolvedModuleViaHttpFallbackInput { reactVersion?: string; dependencyPinningCacheKey?: string; moduleServerOrigin?: string; + serverExternalPackages?: readonly string[]; fetchViaHttp?: FetchModuleViaHttpFn; cacheLocalModule?: CacheLocalModuleFn; } @@ -65,6 +66,7 @@ export async function resolveUnresolvedModuleViaHttpFallback( input.reactVersion, input.dependencyPinningCacheKey, input.moduleServerOrigin, + input.serverExternalPackages, ); } diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/index.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/index.ts index c4ff17bd15..af3b0fd2ef 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/index.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/index.ts @@ -285,6 +285,7 @@ async function doFetchAndCacheModule( effectiveReactVersion, dependencyPinningCacheKey, moduleServerOrigin, + context.serverExternalPackages, ); const cachedPath = await readValidCachedModulePath({ normalizedPath, @@ -317,6 +318,7 @@ async function doFetchAndCacheModule( reactVersion: effectiveReactVersion, dependencyPinningCacheKey, moduleServerOrigin, + serverExternalPackages: context.serverExternalPackages, parentModulePath, }); } @@ -341,6 +343,7 @@ async function doFetchAndCacheModule( contentHash, dependencyPinningCacheKey, moduleServerOrigin, + context.serverExternalPackages, ) : null; @@ -360,6 +363,7 @@ async function doFetchAndCacheModule( projectDir, effectiveReactVersion, log, + context.serverExternalPackages, ) : null; if (distResult?.code) { @@ -375,6 +379,7 @@ async function doFetchAndCacheModule( normalizedPath, projectSlug, reactVersion: context.reactVersion, + serverExternalPackages: context.serverExternalPackages, moduleServerOrigin, dependencyPinningCacheKey, dependencyPinningDependencies: context.dependencyPinningDependencies, @@ -409,6 +414,7 @@ async function doFetchAndCacheModule( reactVersion: effectiveReactVersion, dependencyPinningCacheKey, moduleServerOrigin, + serverExternalPackages: context.serverExternalPackages, distributedCacheWrite: needsDistributedCacheWrite && distResult?.distributedCache && transformCacheKey && contentSourceId @@ -446,6 +452,7 @@ export function createModuleFetcherContext( dependencyPinningCacheKey?: string; dependencyPinningDependencies?: Readonly>; dependencyPinningSource?: ModuleFetcherContext["dependencyPinningSource"]; + serverExternalPackages?: readonly string[]; logger?: Logger; strictMissingModules?: boolean; }, diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/module-cache.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/module-cache.ts index 357b208a22..b5220738eb 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/module-cache.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/module-cache.ts @@ -61,6 +61,7 @@ export async function cacheModule( reactVersion = REACT_DEFAULT_VERSION, dependencyPinningCacheKey = "off", moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise { moduleCode = ensureFilenameDefaultExport(normalizedPath, moduleCode); @@ -78,7 +79,11 @@ export async function cacheModule( const pathCacheKey = buildMdxEsmPathCacheKey( normalizedPath, reactVersion, - getMdxModuleCacheVariant(dependencyPinningCacheKey, moduleServerOrigin), + getMdxModuleCacheVariant( + dependencyPinningCacheKey, + moduleServerOrigin, + serverExternalPackages, + ), ); const localFs = getLocalFs(); diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/persistence.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/persistence.ts index 8cbd9e9d34..89dbff6aef 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/persistence.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/persistence.ts @@ -24,6 +24,7 @@ export interface PersistResolvedModuleInput { reactVersion?: string; dependencyPinningCacheKey?: string; moduleServerOrigin?: string; + serverExternalPackages?: readonly string[]; distributedCacheWrite?: { distributedCache: DistributedCache; transformCacheKey: string; @@ -70,6 +71,7 @@ export async function persistResolvedModule( input.reactVersion, input.dependencyPinningCacheKey, input.moduleServerOrigin, + input.serverExternalPackages, ); input.log.debug(`${LOG_PREFIX_MDX_LOADER} [fetchAndCacheModule] cacheModule DONE`, { projectSlug: input.projectSlug, diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.test.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.test.ts index 7cf13a92ae..9551b4e1d8 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.test.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.test.ts @@ -24,6 +24,7 @@ describe("module-fetcher/source-transform", () => { normalizedPath: "_vf_modules/app/page.tsx", projectSlug: "docs", reactVersion: "19.1.1", + serverExternalPackages: ["knex"], moduleServerOrigin: "https://preview.example", dependencyPinningCacheKey: "on:pins", adapter, @@ -43,6 +44,7 @@ describe("module-fetcher/source-transform", () => { dev: true, ssr: true, reactVersion: "19.1.1", + serverExternalPackages: ["knex"], moduleServerOrigin: "https://preview.example", dependencyPinningCacheKey: "on:pins", }); @@ -57,6 +59,7 @@ describe("module-fetcher/source-transform", () => { calls.push("cacheHttpImportsToLocal"); assertEquals(code, `import React from "https://esm.sh/react";\nexport default React;`); assertEquals(options.reactVersion, "19.1.1"); + assertEquals(options.serverExternalPackages, ["knex"]); return Promise.resolve({ code: `import React from "file:///cache/react.mjs";\nexport default React;`, }); diff --git a/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.ts b/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.ts index f8fc6ef963..b83a37709f 100644 --- a/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.ts +++ b/src/transforms/mdx/esm-module-loader/module-fetcher/source-transform.ts @@ -27,6 +27,7 @@ export interface TransformResolvedModuleSourceInput { normalizedPath: string; projectSlug: string; reactVersion?: string; + serverExternalPackages?: readonly string[]; moduleServerOrigin?: string; dependencyPinningCacheKey?: string; dependencyPinningDependencies?: Readonly>; @@ -66,6 +67,7 @@ export async function transformResolvedModuleSource( dev: true, ssr: true, reactVersion: input.reactVersion, + serverExternalPackages: input.serverExternalPackages, moduleServerOrigin: input.moduleServerOrigin, dependencyPinningCacheKey: input.dependencyPinningCacheKey, ...(input.dependencyPinningDependencies === undefined @@ -106,6 +108,7 @@ export async function transformResolvedModuleSource( cacheDir: getHttpBundleCacheDir(), importMap, reactVersion: input.reactVersion, + serverExternalPackages: input.serverExternalPackages, }); return cacheResult.code; diff --git a/src/transforms/mdx/esm-module-loader/module-writer.test.ts b/src/transforms/mdx/esm-module-loader/module-writer.test.ts index b625bf43d9..dc26b50f42 100644 --- a/src/transforms/mdx/esm-module-loader/module-writer.test.ts +++ b/src/transforms/mdx/esm-module-loader/module-writer.test.ts @@ -96,6 +96,37 @@ describe("MDX root module cache identity", () => { assertEquals(originA.compositeKey === originB.compositeKey, false); }); + it("isolates root modules by the configured server external package set", async () => { + const code = "export const value = 1;"; + const baseline = await buildMdxModuleCacheIdentity( + "/cache/mdx", + "project-id", + "19.1.1", + code, + ); + const combined = await buildMdxModuleCacheIdentity( + "/cache/mdx", + "project-id", + "19.1.1", + code, + "off", + undefined, + ["knex", "@prisma/client"], + ); + const reordered = await buildMdxModuleCacheIdentity( + "/cache/mdx", + "project-id", + "19.1.1", + code, + "off", + undefined, + ["@prisma/client", "knex"], + ); + + assertEquals(combined.namespaceKey === baseline.namespaceKey, false); + assertEquals(reordered, combined); + }); + it("pins a top-level same-origin absolute module before strict SSR fetching", async () => { const modulePath = `/_vf_modules/StrictChild-${crypto.randomUUID()}.js`; const dependencies = {}; diff --git a/src/transforms/mdx/esm-module-loader/module-writer.ts b/src/transforms/mdx/esm-module-loader/module-writer.ts index de43b573ec..7809a37e34 100644 --- a/src/transforms/mdx/esm-module-loader/module-writer.ts +++ b/src/transforms/mdx/esm-module-loader/module-writer.ts @@ -56,6 +56,7 @@ import { } from "./loader-helpers.ts"; import { hasUnresolvedImports } from "./module-fetcher/nested-imports.ts"; import { resolveDependencyPinningSnapshot } from "#veryfront/transforms/esm/package-registry.ts"; +import { buildServerExternalPackagesIdentity } from "#veryfront/config/server-external-packages.ts"; /** Singleflight for MDX module file writes to prevent race conditions */ const mdxWriteFlight = new Singleflight(); @@ -65,6 +66,7 @@ export function buildMdxModuleNamespaceKey( reactVersion: string, dependencyPinningCacheKey?: string, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise { const pinIdentity = dependencyPinningCacheKey?.startsWith("on:") ? `:pins-${dependencyPinningCacheKey}` @@ -72,7 +74,15 @@ export function buildMdxModuleNamespaceKey( const originIdentity = dependencyPinningCacheKey?.startsWith("on:") && moduleServerOrigin ? `:origin-${moduleServerOrigin}` : ""; - return computeHash(`${projectId}:react-${reactVersion}${pinIdentity}${originIdentity}`); + const serverExternalPackagesIdentity = buildServerExternalPackagesIdentity( + serverExternalPackages, + ); + const externalIdentity = serverExternalPackagesIdentity + ? `:server-externals-${serverExternalPackagesIdentity}` + : ""; + return computeHash( + `${projectId}:react-${reactVersion}${pinIdentity}${originIdentity}${externalIdentity}`, + ); } export async function buildMdxModuleCacheIdentity( @@ -82,6 +92,7 @@ export async function buildMdxModuleCacheIdentity( rewrittenCode: string, dependencyPinningCacheKey?: string, moduleServerOrigin?: string, + serverExternalPackages?: readonly string[], ): Promise<{ namespaceKey: string; codeHash: string; @@ -95,6 +106,7 @@ export async function buildMdxModuleCacheIdentity( reactVersion, dependencyPinningCacheKey, moduleServerOrigin, + serverExternalPackages, ); const codeHash = hashString(rewrittenCode); const namespaceDir = join(esmCacheDir, namespaceKey); @@ -125,11 +137,13 @@ async function cacheHttpImports( code: string, importMap: ImportMapConfig, reactVersion?: string, + serverExternalPackages?: readonly string[], ): Promise { const result = await cacheHttpImportsToLocal(code, { cacheDir: getHttpBundleCacheDir(), importMap, reactVersion, + serverExternalPackages, }); return result.code; } @@ -226,6 +240,7 @@ export async function doLoadModuleESM( dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, dependencyPinningSource, + serverExternalPackages: effectiveContext.serverExternalPackages, }); rewritten = await pinSameOriginSSRModuleImports( rewritten, @@ -265,7 +280,13 @@ export async function doLoadModuleESM( logger.debug(`${LOG_PREFIX_MDX_LOADER} Step: cacheHttpImports START`, { projectSlug }); rewritten = await withSpan( SpanNames.MDX_CACHE_HTTP, - () => cacheHttpImports(rewritten, importMap, effectiveContext.reactVersion), + () => + cacheHttpImports( + rewritten, + importMap, + effectiveContext.reactVersion, + effectiveContext.serverExternalPackages, + ), { "mdx.project_slug": projectSlug }, ); logger.debug(`${LOG_PREFIX_MDX_LOADER} Step: cacheHttpImports DONE`, { projectSlug }); @@ -284,6 +305,7 @@ export async function doLoadModuleESM( rewritten, dependencySnapshot.cacheKey, effectiveContext.moduleServerOrigin, + effectiveContext.serverExternalPackages, ); const namespaceKey = cacheIdentity.namespaceKey; let codeHash = cacheIdentity.codeHash; @@ -375,6 +397,7 @@ export async function doLoadModuleESM( cacheDir, importMap, reactVersion: effectiveContext.reactVersion, + serverExternalPackages: effectiveContext.serverExternalPackages, }); rewritten = refreshResult.code; diff --git a/src/transforms/mdx/esm-module-loader/types.ts b/src/transforms/mdx/esm-module-loader/types.ts index f485d5b6bb..c4fa05bed7 100644 --- a/src/transforms/mdx/esm-module-loader/types.ts +++ b/src/transforms/mdx/esm-module-loader/types.ts @@ -17,6 +17,8 @@ export interface ESMLoaderContext { isLocalProject?: boolean; /** React version for transforms (from project config) */ reactVersion?: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Absolute request origin used to identify same-origin module URLs. */ moduleServerOrigin?: string; /** Request-scoped dependency-pinning state used to isolate module caches. */ @@ -84,6 +86,8 @@ export interface ModuleFetcherContext { moduleGraph?: Set; /** React version for transforms (from project config) */ reactVersion?: string; + /** Bare npm package roots that the runtime resolves without bundling. */ + serverExternalPackages?: readonly string[]; /** Absolute request origin used to identify same-origin module URLs. */ moduleServerOrigin?: string; /** Request-scoped dependency-pinning state used to isolate module caches. */ diff --git a/src/transforms/mdx/index.ts b/src/transforms/mdx/index.ts index 288510e6ce..2616457455 100644 --- a/src/transforms/mdx/index.ts +++ b/src/transforms/mdx/index.ts @@ -32,6 +32,7 @@ export interface MDXLoadModuleOptions { projectSlug?: string; contentSourceId?: string; reactVersion?: string; + serverExternalPackages?: readonly string[]; dependencyPinningCacheKey?: string; dependencyPinningDependencies?: Readonly>; dependencyPinningSource?: DependencyPinningSourceInput; @@ -106,6 +107,7 @@ export class MDXRenderer { projectSlug, contentSourceId, reactVersion, + serverExternalPackages, dependencyPinningCacheKey, dependencyPinningDependencies, dependencyPinningSource, @@ -127,6 +129,7 @@ export class MDXRenderer { projectSlug, contentSourceId, reactVersion, + serverExternalPackages, dependencyPinningCacheKey: dependencySnapshot.cacheKey, dependencyPinningDependencies: dependencySnapshot.dependencies, dependencyPinningSource: resolvedDependencyPinningSource, diff --git a/src/transforms/pipeline/cache-identity.test.ts b/src/transforms/pipeline/cache-identity.test.ts index 99501d4b21..716e1ab66e 100644 --- a/src/transforms/pipeline/cache-identity.test.ts +++ b/src/transforms/pipeline/cache-identity.test.ts @@ -321,6 +321,23 @@ describe("transform pipeline cache identity", () => { assertNotEquals(changed, baseline); }); + it("partitions transforms by the configured server external package set", async () => { + const baseline = await computePipelineConfigIdentity(identityInput()); + const knex = await computePipelineConfigIdentity( + identityInput({ serverExternalPackages: ["knex"] }), + ); + const prismaAndKnex = await computePipelineConfigIdentity( + identityInput({ serverExternalPackages: ["@prisma/client", "knex"] }), + ); + const reordered = await computePipelineConfigIdentity( + identityInput({ serverExternalPackages: ["knex", "@prisma/client"] }), + ); + + assertNotEquals(knex, baseline); + assertNotEquals(prismaAndKnex, knex); + assertEquals(reordered, prismaAndKnex); + }); + it("keeps ill-formed Unicode distinct from replacement characters", async () => { const loneSurrogate = await computePipelineConfigIdentity( identityInput({ projectDir: "\ud800" }), diff --git a/src/transforms/pipeline/cache-identity.ts b/src/transforms/pipeline/cache-identity.ts index d1772e03a8..ba7a24ad70 100644 --- a/src/transforms/pipeline/cache-identity.ts +++ b/src/transforms/pipeline/cache-identity.ts @@ -3,6 +3,7 @@ import { computeConfigHash } from "#veryfront/cache/config-hash.ts"; import { fingerprintImportMap } from "../esm/http-cache-helpers.ts"; import type { ImportMapConfig } from "#veryfront/modules/import-map/types.ts"; import type { TransformPlugin } from "./types.ts"; +import { canonicalizeServerExternalPackages } from "#veryfront/config/server-external-packages.ts"; const MAX_IMPORT_MAP_ENTRIES = 20_000; const MAX_IDENTITY_STRING_BYTES = 64 * 1024; @@ -450,6 +451,7 @@ export interface PipelineConfigIdentityInput { apiBaseUrl?: string; importMapFingerprint?: string; dependencyPinningCacheKey?: string; + serverExternalPackages?: readonly string[]; customPlugins: ReadonlyArray; } @@ -533,6 +535,26 @@ export async function computePipelineConfigIdentity( identity += `api=${encodeIdentityPrimitive(apiBaseUrl)};`; identity += `import-map=${encodeIdentityPrimitive(importMapFingerprint)};`; identity += `dependency-pins=${encodeIdentityPrimitive(dependencyPinningCacheKey)};`; + const serverExternalPackages = readOwnDataProperty( + input, + "serverExternalPackages", + "Transform pipeline identity", + ); + if (serverExternalPackages !== undefined) { + if (!ArrayIsArray(serverExternalPackages)) { + throw new IntrinsicTypeError("Server external packages must be an array"); + } + const canonical = canonicalizeServerExternalPackages( + serverExternalPackages as readonly string[], + ); + if (canonical) { + identity += "server-externals="; + for (let index = 0; index < canonical.length; index++) { + identity += `${encodeIdentityPrimitive(canonical[index]!)},`; + } + identity += ";"; + } + } identity += `plugins=${customPlugins}`; return computeHash(identity); } diff --git a/src/transforms/pipeline/context.ts b/src/transforms/pipeline/context.ts index ba41ec3090..eb13b58a17 100644 --- a/src/transforms/pipeline/context.ts +++ b/src/transforms/pipeline/context.ts @@ -6,6 +6,7 @@ import type { TransformStage, TransformTarget, } from "./types.ts"; +import { canonicalizeServerExternalPackages } from "#veryfront/config/server-external-packages.ts"; function buildContext( source: string, @@ -36,6 +37,7 @@ function buildContext( metadata: new Map(), studioEmbed: options.studioEmbed, reactVersion, + serverExternalPackages: canonicalizeServerExternalPackages(options.serverExternalPackages), dependencyPinningCacheKey: options.dependencyPinningCacheKey, dependencyPinningDependencies: options.dependencyPinningDependencies, dependencyPinningSource: options.dependencyPinningSource, diff --git a/src/transforms/pipeline/index.ts b/src/transforms/pipeline/index.ts index eda262e381..d76222c8c5 100644 --- a/src/transforms/pipeline/index.ts +++ b/src/transforms/pipeline/index.ts @@ -246,6 +246,7 @@ export function runPipeline( projectDir, importMapFingerprint, dependencyPinningCacheKey, + serverExternalPackages: ctx.serverExternalPackages, customPlugins: pluginCacheIdentity.identity, }), computeDepsHashSafe( diff --git a/src/transforms/pipeline/stages/resolve-imports.ts b/src/transforms/pipeline/stages/resolve-imports.ts index c38edf8a26..040e9c0ecc 100644 --- a/src/transforms/pipeline/stages/resolve-imports.ts +++ b/src/transforms/pipeline/stages/resolve-imports.ts @@ -47,6 +47,7 @@ async function buildRewriteContext(ctx: TransformContext): Promise { - for (const pkg of ["redis", "ioredis", "pg", "mysql2", "better-sqlite3", "mongodb"]) { - assertEquals(isServerOnlyPackage(pkg), true, `${pkg} should be server-only`); - } -}); +describe("isServerOnlyPackage", () => { + it("recognizes known server-only drivers", () => { + for (const pkg of ["redis", "ioredis", "pg", "mysql2", "better-sqlite3", "mongodb"]) { + assertEquals(isServerOnlyPackage(pkg), true, `${pkg} should be server-only`); + } + }); -Deno.test("isServerOnlyPackage: strips an npm: prefix before matching", () => { - assertEquals(isServerOnlyPackage("npm:redis"), true); -}); + it("strips an npm: prefix before matching", () => { + assertEquals(isServerOnlyPackage("npm:redis"), true); + }); + + it("recognizes configured package names", () => { + const configured = ["knex", "@prisma/client"]; + + assertEquals(isServerOnlyPackage("knex", configured), true); + assertEquals(isServerOnlyPackage("npm:knex", configured), true); + assertEquals(isServerOnlyPackage("@prisma/client", configured), true); + }); -Deno.test("isServerOnlyPackage: leaves browser-safe packages alone", () => { - for (const pkg of ["react", "react-dom", "zod", "lodash", "@tanstack/react-query"]) { - assertEquals(isServerOnlyPackage(pkg), false, `${pkg} should not be server-only`); - } + it("leaves browser-safe packages alone", () => { + for (const pkg of ["react", "react-dom", "zod", "lodash", "@tanstack/react-query"]) { + assertEquals(isServerOnlyPackage(pkg), false, `${pkg} should not be server-only`); + } + }); }); diff --git a/src/transforms/shared/server-only-packages.ts b/src/transforms/shared/server-only-packages.ts index e14ef57c35..a3460b6e56 100644 --- a/src/transforms/shared/server-only-packages.ts +++ b/src/transforms/shared/server-only-packages.ts @@ -34,6 +34,7 @@ const SERVER_ONLY_PACKAGES: ReadonlySet = new Set([ ]); const ReflectApply = Reflect.apply; const SetHas = Set.prototype.has; +const ArrayIncludes = Array.prototype.includes; const StringSlice = String.prototype.slice; const StringStartsWith = String.prototype.startsWith; @@ -57,11 +58,16 @@ function stringStartsWith(value: string, search: string): boolean { * carry an `npm:` prefix, e.g. `npm:redis`). The `npm:` prefix is stripped * before matching so both `redis` and `npm:redis@5.11.0` are recognized. */ -export function isServerOnlyPackage(packageName: string): boolean { +export function isServerOnlyPackage( + packageName: string, + configuredPackages?: readonly string[], +): boolean { const bare = stringStartsWith(packageName, "npm:") ? stringSlice(packageName, "npm:".length) : packageName; - return setHas(SERVER_ONLY_PACKAGES, bare); + return setHas(SERVER_ONLY_PACKAGES, bare) || + (configuredPackages !== undefined && + ReflectApply(ArrayIncludes, configuredPackages, [bare]) as boolean); } export { SERVER_ONLY_PACKAGES };