Skip to content

Commit

Permalink
feat: allow importing CSS and assets inside external dependencies (#3880
Browse files Browse the repository at this point in the history
)
  • Loading branch information
sheremet-va authored Aug 3, 2023
1 parent 47f5c3a commit f4e6e99
Show file tree
Hide file tree
Showing 24 changed files with 444 additions and 202 deletions.
57 changes: 56 additions & 1 deletion docs/config/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -196,14 +196,69 @@ When Vitest encounters the external library listed in `include`, it will be bund
- Your `alias` configuration is now respected inside bundled packages
- Code in your tests is running closer to how it's running in the browser

Be aware that only packages in `deps.optimizer?.[mode].include` option are bundled (some plugins populate this automatically, like Svelte). You can read more about available options in [Vite](https://vitejs.dev/config/dep-optimization-options.html) docs. By default, Vitest uses `optimizer.web` for `jsdom` and `happy-dom` environments, and `optimizer.ssr` for `node` and `edge` environments, but it is configurable by [`transformMode`](#transformmode).
Be aware that only packages in `deps.optimizer?.[mode].include` option are bundled (some plugins populate this automatically, like Svelte). You can read more about available options in [Vite](https://vitejs.dev/config/dep-optimization-options.html) docs (Vitest doesn't support `disable` and `noDiscovery` options). By default, Vitest uses `optimizer.web` for `jsdom` and `happy-dom` environments, and `optimizer.ssr` for `node` and `edge` environments, but it is configurable by [`transformMode`](#transformmode).

This options also inherits your `optimizeDeps` configuration (for web Vitest will extend `optimizeDeps`, for ssr - `ssr.optimizeDeps`). If you redefine `include`/`exclude` option in `deps.optimizer` it will extend your `optimizeDeps` when running tests. Vitest automatically removes the same options from `include`, if they are listed in `exclude`.

::: tip
You will not be able to edit your `node_modules` code for debugging, since the code is actually located in your `cacheDir` or `test.cache.dir` directory. If you want to debug with `console.log` statements, edit it directly or force rebundling with `deps.optimizer?.[mode].force` option.
:::

#### deps.optimizer.{mode}.enabled

- **Type:** `boolean`
- **Default:** `true`

Enable dependency optimization.

#### deps.web

- **Type:** `{ transformAssets?, ... }`
- **Version:** Since Vite 0.34.2

Options that are applied to external files when transform mode is set to `web`. By default, `jsdom` and `happy-dom` use `web` mode, while `node` and `edge` environments use `ssr` transform mode, so these options will have no affect on files inside those environments.

Usually, files inside `node_modules` are externalized, but these options also affect files in [`server.deps.external`](#server-deps-external).

#### deps.web.transformAssets

- **Type:** `boolean`
- **Default:** `true`

Should Vitest process assets (.png, .svg, .jpg, etc) files and resolve them like Vite does in the browser.

hese module will have a default export equal to the path to the asset, if no query is specified.

::: warning
At the moment, this option only works with [`experimentalVmThreads`](#experimentalvmthreads) pool.
:::

#### deps.web.transformCss

- **Type:** `boolean`
- **Default:** `true`

Should Vitest process CSS (.css, .scss, .sass, etc) files and resolve them like Vite does in the browser.

If CSS files are disabled with [`css`](#css) options, this option will just silence `UNKNOWN_EXTENSION` errors.

::: warning
At the moment, this option only works with [`experimentalVmThreads`](#experimentalvmthreads) pool.
:::

#### deps.web.transformGlobPattern

- **Type:** `RegExp | RegExp[]`
- **Default:** `[]`

Regexp pattern to match external files that should be transformed.

By default, files inside `node_modules` are externalized and not transformed, unless it's CSS or an asset, and corresponding option is not disabled.

::: warning
At the moment, this option only works with [`experimentalVmThreads`](#experimentalvmthreads) pool.
:::

#### deps.registerNodeLoader<NonProjectOption />

- **Type:** `boolean`
Expand Down
13 changes: 13 additions & 0 deletions packages/vite-node/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,19 @@ export class ViteNodeServer {
return this.transformPromiseMap.get(id)!
}

async transformModule(id: string, transformMode?: 'web' | 'ssr') {
if (transformMode !== 'web')
throw new Error('`transformModule` only supports `transformMode: "web"`.')

const normalizedId = normalizeModuleId(id)
const mod = this.server.moduleGraph.getModuleById(normalizedId)
const result = mod?.transformResult || await this.server.transformRequest(normalizedId)

return {
code: result?.code,
}
}

getTransformMode(id: string) {
const withoutQuery = id.split('?')[0]

Expand Down
5 changes: 5 additions & 0 deletions packages/vitest/src/node/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,11 @@ export function resolveConfig(
if (!resolved.deps.moduleDirectories.includes('/node_modules/'))
resolved.deps.moduleDirectories.push('/node_modules/')

resolved.deps.web ??= {}
resolved.deps.web.transformAssets ??= true
resolved.deps.web.transformCss ??= true
resolved.deps.web.transformGlobPattern ??= []

resolved.server ??= {}
resolved.server.deps ??= {}

Expand Down
3 changes: 3 additions & 0 deletions packages/vitest/src/node/pools/rpc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ export function createMethodsRPC(project: WorkspaceProject): RuntimeRPC {
resolveId(id, importer, transformMode) {
return project.vitenode.resolveId(id, importer, transformMode)
},
transform(id, environment) {
return project.vitenode.transformModule(id, environment)
},
onPathsCollected(paths) {
ctx.state.collectPaths(paths)
project.report('onPathsCollected', paths)
Expand Down
4 changes: 2 additions & 2 deletions packages/vitest/src/node/pools/vm-threads.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import type { WorkspaceProject } from '../workspace'
import { createMethodsRPC } from './rpc'

const workerPath = pathToFileURL(resolve(distDir, './vm.js')).href
const suppressLoaderWarningsPath = resolve(rootDir, './suppress-warnings.cjs')
const suppressWarningsPath = resolve(rootDir, './suppress-warnings.cjs')

function createWorkerChannel(project: WorkspaceProject) {
const channel = new MessageChannel()
Expand Down Expand Up @@ -61,7 +61,7 @@ export function createVmThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessO
'--experimental-import-meta-resolve',
'--experimental-vm-modules',
'--require',
suppressLoaderWarningsPath,
suppressWarningsPath,
...execArgv,
],

Expand Down
4 changes: 2 additions & 2 deletions packages/vitest/src/node/workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -293,10 +293,10 @@ export class WorkspaceProject {
...this.config.deps,
optimizer: {
web: {
enabled: this.config.deps?.optimizer?.web?.enabled ?? false,
enabled: this.config.deps?.optimizer?.web?.enabled ?? true,
},
ssr: {
enabled: this.config.deps?.optimizer?.ssr?.enabled ?? false,
enabled: this.config.deps?.optimizer?.ssr?.enabled ?? true,
},
},
},
Expand Down
18 changes: 11 additions & 7 deletions packages/vitest/src/runtime/execute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import type { ViteNodeRunnerOptions } from 'vite-node'
import { normalize, relative, resolve } from 'pathe'
import { processError } from '@vitest/utils/error'
import type { MockMap } from '../types/mocker'
import type { ResolvedConfig, ResolvedTestEnvironment, WorkerGlobalState } from '../types'
import type { ResolvedConfig, ResolvedTestEnvironment, RuntimeRPC, WorkerGlobalState } from '../types'
import { distDir } from '../paths'
import { getWorkerState } from '../utils/global'
import { VitestMocker } from './mocker'
Expand All @@ -21,6 +21,7 @@ export interface ExecuteOptions extends ViteNodeRunnerOptions {
moduleDirectories?: string[]
context?: vm.Context
state: WorkerGlobalState
transform: RuntimeRPC['transform']
}

export async function createVitestExecutor(options: ExecuteOptions) {
Expand Down Expand Up @@ -100,6 +101,9 @@ export async function startVitestExecutor(options: ContextExecutorOptions) {
resolveId(id, importer) {
return rpc().resolveId(id, importer, getTransformMode())
},
transform(id) {
return rpc().transform(id, 'web')
},
packageCache,
moduleCache,
mockMap,
Expand Down Expand Up @@ -174,12 +178,6 @@ export class VitestExecutor extends ViteNodeRunner {
}
}
else {
this.externalModules = new ExternalModulesExecutor({
...options,
fileMap,
context: options.context,
packageCache: options.packageCache,
})
const clientStub = vm.runInContext(
`(defaultClient) => ({ ...defaultClient, updateStyle: ${updateStyle.toString()}, removeStyle: ${removeStyle.toString()} })`,
options.context,
Expand All @@ -189,6 +187,12 @@ export class VitestExecutor extends ViteNodeRunner {
'@vite/client': clientStub,
}
this.primitives = vm.runInContext('({ Object, Reflect, Symbol })', options.context)
this.externalModules = new ExternalModulesExecutor({
...options,
fileMap,
context: options.context,
packageCache: options.packageCache,
})
}
}

Expand Down
92 changes: 64 additions & 28 deletions packages/vitest/src/runtime/external-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { CommonjsExecutor } from './vm/commonjs-executor'
import type { FileMap } from './vm/file-map'
import { EsmExecutor } from './vm/esm-executor'
import { interopCommonJsModule } from './vm/utils'
import { ViteExecutor } from './vm/vite-executor'

const SyntheticModule: typeof VMSyntheticModule = (vm as any).SyntheticModule

Expand All @@ -23,12 +24,20 @@ export interface ExternalModulesExecutorOptions extends ExecuteOptions {
packageCache: Map<string, any>
}

interface ModuleInformation {
type: 'data' | 'builtin' | 'vite' | 'module' | 'commonjs'
url: string
path: string
}

// TODO: improve Node.js strict mode support in #2854
export class ExternalModulesExecutor {
private cjs: CommonjsExecutor
private esm: EsmExecutor
private vite: ViteExecutor
private context: vm.Context
private fs: FileMap
private resolvers: ((id: string, parent: string) => string | undefined)[] = []

constructor(private options: ExternalModulesExecutorOptions) {
this.context = options.context
Expand All @@ -42,6 +51,13 @@ export class ExternalModulesExecutor {
importModuleDynamically: this.importModuleDynamically,
fileMap: options.fileMap,
})
this.vite = new ViteExecutor({
esmExecutor: this.esm,
context: this.context,
transform: options.transform,
viteClientModule: options.requestStubs!['/@vite/client'],
})
this.resolvers = [this.vite.resolve]
}

// dynamic import can be used in both ESM and CJS, so we have it in the executor
Expand All @@ -56,6 +72,11 @@ export class ExternalModulesExecutor {
}

public async resolve(specifier: string, parent: string) {
for (const resolver of this.resolvers) {
const id = resolver(specifier, parent)
if (id)
return id
}
return nativeResolve(specifier, parent)
}

Expand Down Expand Up @@ -124,44 +145,59 @@ export class ExternalModulesExecutor {
return m
}

private async createModule(identifier: string): Promise<VMModule> {
private getModuleInformation(identifier: string): ModuleInformation {
if (identifier.startsWith('data:'))
return this.esm.createDataModule(identifier)
return { type: 'data', url: identifier, path: identifier }

const extension = extname(identifier)

if (extension === '.node' || isNodeBuiltin(identifier)) {
const exports = this.require(identifier)
return this.wrapCoreSynteticModule(identifier, exports)
}
if (extension === '.node' || isNodeBuiltin(identifier))
return { type: 'builtin', url: identifier, path: identifier }

const isFileUrl = identifier.startsWith('file://')
const fileUrl = isFileUrl ? identifier : pathToFileURL(identifier).toString()
const pathUrl = isFileUrl ? fileURLToPath(identifier.split('?')[0]) : identifier
const fileUrl = isFileUrl ? identifier : pathToFileURL(pathUrl).toString()

// TODO: support wasm in the future
// if (extension === '.wasm') {
// const source = this.readBuffer(pathUrl)
// const wasm = this.loadWebAssemblyModule(source, fileUrl)
// this.moduleCache.set(fileUrl, wasm)
// return wasm
// }

if (extension === '.cjs') {
const exports = this.require(pathUrl)
return this.wrapCommonJsSynteticModule(fileUrl, exports)
let type: 'module' | 'commonjs' | 'vite'
if (this.vite.canResolve(fileUrl)) {
type = 'vite'
}
else if (extension === '.mjs') {
type = 'module'
}
else if (extension === '.cjs') {
type = 'commonjs'
}
else {
const pkgData = this.findNearestPackageData(normalize(pathUrl))
type = pkgData.type === 'module' ? 'module' : 'commonjs'
}

if (extension === '.mjs')
return await this.esm.createEsmModule(fileUrl, this.fs.readFile(pathUrl))

const pkgData = this.findNearestPackageData(normalize(pathUrl))

if (pkgData.type === 'module')
return await this.esm.createEsmModule(fileUrl, this.fs.readFile(pathUrl))
return { type, path: pathUrl, url: fileUrl }
}

const exports = this.cjs.require(pathUrl)
return this.wrapCommonJsSynteticModule(fileUrl, exports)
private async createModule(identifier: string): Promise<VMModule> {
const { type, url, path } = this.getModuleInformation(identifier)

switch (type) {
case 'data':
return this.esm.createDataModule(identifier)
case 'builtin': {
const exports = this.require(identifier)
return this.wrapCoreSynteticModule(identifier, exports)
}
case 'vite':
return await this.vite.createViteModule(url)
case 'module':
return await this.esm.createEsModule(url, this.fs.readFile(path))
case 'commonjs': {
const exports = this.require(path)
return this.wrapCommonJsSynteticModule(identifier, exports)
}
default: {
const _deadend: never = type
return _deadend
}
}
}

async import(identifier: string) {
Expand Down
28 changes: 23 additions & 5 deletions packages/vitest/src/runtime/vm/commonjs-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ interface PrivateNodeModule extends NodeModule {

export class CommonjsExecutor {
private context: vm.Context
private requireCache: Record<string, NodeModule> = Object.create(null)
private requireCache = new Map<string, NodeModule>()
private publicRequireCache = this.createProxyCache()

private moduleCache = new Map<string, VMModule | Promise<VMModule>>()
private builtinCache: Record<string, NodeModule> = Object.create(null)
Expand Down Expand Up @@ -75,7 +76,7 @@ export class CommonjsExecutor {
script.identifier = filename
const fn = script.runInContext(executor.context)
const __dirname = dirname(filename)
executor.requireCache[filename] = this
executor.requireCache.set(filename, this)
try {
fn(this.exports, this.require, this, filename, __dirname)
return this.exports
Expand Down Expand Up @@ -165,14 +166,31 @@ export class CommonjsExecutor {
set: () => {},
configurable: true,
})
require.main = _require.main
require.cache = this.requireCache
require.main = undefined // there is no main, since we are running tests using ESM
require.cache = this.publicRequireCache
return require
}

private createProxyCache() {
return new Proxy(Object.create(null), {
defineProperty: () => true,
deleteProperty: () => true,
set: () => true,
get: (_, key: string) => this.requireCache.get(key),
has: (_, key: string) => this.requireCache.has(key),
ownKeys: () => Array.from(this.requireCache.keys()),
getOwnPropertyDescriptor() {
return {
configurable: true,
enumerable: true,
}
},
})
}

// very naive implementation for Node.js require
private loadCommonJSModule(module: NodeModule, filename: string): Record<string, unknown> {
const cached = this.requireCache[filename]
const cached = this.requireCache.get(filename)
if (cached)
return cached.exports

Expand Down
Loading

0 comments on commit f4e6e99

Please sign in to comment.