From 103a6002759b5e73cf07a41fb08512bd4248c4c1 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 30 Apr 2024 12:02:54 +0200 Subject: [PATCH] fix(vm): support network imports (#5610) --- .../vitest/src/runtime/external-executor.ts | 50 +++++++++++++------ .../vitest/src/runtime/vm/esm-executor.ts | 43 ++++++++++++++-- .../vitest/src/runtime/vm/vite-executor.ts | 10 ++-- .../fixtures/network-imports/basic.test.ts | 7 +++ test/cli/test/network-imports.test.ts | 7 +-- 5 files changed, 88 insertions(+), 29 deletions(-) diff --git a/packages/vitest/src/runtime/external-executor.ts b/packages/vitest/src/runtime/external-executor.ts index 81aec41db0e9..18c5354cef1c 100644 --- a/packages/vitest/src/runtime/external-executor.ts +++ b/packages/vitest/src/runtime/external-executor.ts @@ -27,7 +27,7 @@ export interface ExternalModulesExecutorOptions { } interface ModuleInformation { - type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs' + type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs' | 'network' url: string path: string } @@ -41,6 +41,8 @@ export class ExternalModulesExecutor { private fs: FileMap private resolvers: ((id: string, parent: string) => string | undefined)[] = [] + #networkSupported: boolean | null = null + constructor(private options: ExternalModulesExecutorOptions) { this.context = options.context @@ -62,6 +64,20 @@ export class ExternalModulesExecutor { this.resolvers = [this.vite.resolve] } + async import(identifier: string) { + const module = await this.createModule(identifier) + await this.esm.evaluateModule(module) + return module.namespace + } + + require(identifier: string) { + return this.cjs.require(identifier) + } + + createRequire(identifier: string) { + return this.cjs.createRequire(identifier) + } + // dynamic import can be used in both ESM and CJS, so we have it in the executor public importModuleDynamically = async (specifier: string, referencer: VMModule) => { const module = await this.resolveModule(specifier, referencer.identifier) @@ -161,6 +177,9 @@ export class ExternalModulesExecutor { if (extension === '.node' || isNodeBuiltin(identifier)) return { type: 'builtin', url: identifier, path: identifier } + if (this.isNetworkSupported && (identifier.startsWith('http:') || identifier.startsWith('https:'))) + return { type: 'network', url: identifier, path: identifier } + const isFileUrl = identifier.startsWith('file://') const pathUrl = isFileUrl ? fileURLToPath(identifier.split('?')[0]) : identifier const fileUrl = isFileUrl ? identifier : pathToFileURL(pathUrl).toString() @@ -209,13 +228,16 @@ export class ExternalModulesExecutor { case 'vite': return await this.vite.createViteModule(url) case 'wasm': - return await this.esm.createWebAssemblyModule(url, this.fs.readBuffer(path)) + return await this.esm.createWebAssemblyModule(url, () => this.fs.readBuffer(path)) case 'module': - return await this.esm.createEsModule(url, this.fs.readFile(path)) + return await this.esm.createEsModule(url, () => this.fs.readFile(path)) case 'commonjs': { const exports = this.require(path) return this.wrapCommonJsSynteticModule(identifier, exports) } + case 'network': { + return this.esm.createNetworkModule(url) + } default: { const _deadend: never = type return _deadend @@ -223,17 +245,15 @@ export class ExternalModulesExecutor { } } - async import(identifier: string) { - const module = await this.createModule(identifier) - await this.esm.evaluateModule(module) - return module.namespace - } - - require(identifier: string) { - return this.cjs.require(identifier) - } - - createRequire(identifier: string) { - return this.cjs.createRequire(identifier) + private get isNetworkSupported() { + if (this.#networkSupported == null) { + if (process.execArgv.includes('--experimental-network-imports')) + this.#networkSupported = true + else if (process.env.NODE_OPTIONS?.includes('--experimental-network-imports')) + this.#networkSupported = true + else + this.#networkSupported = false + } + return this.#networkSupported } } diff --git a/packages/vitest/src/runtime/vm/esm-executor.ts b/packages/vitest/src/runtime/vm/esm-executor.ts index cd429aa538e1..bdc2de826c65 100644 --- a/packages/vitest/src/runtime/vm/esm-executor.ts +++ b/packages/vitest/src/runtime/vm/esm-executor.ts @@ -18,6 +18,8 @@ export class EsmExecutor { private esmLinkMap = new WeakMap>() private context: vm.Context + #httpIp = IPnumber('127.0.0.0') + constructor(private executor: ExternalModulesExecutor, options: EsmExecutorOptions) { this.context = options.context } @@ -38,10 +40,11 @@ export class EsmExecutor { return m } - public async createEsModule(fileUrl: string, code: string) { + public async createEsModule(fileUrl: string, getCode: () => Promise | string) { const cached = this.moduleCache.get(fileUrl) if (cached) return cached + const code = await getCode() // TODO: should not be allowed in strict mode, implement in #2854 if (fileUrl.endsWith('.json')) { const m = new SyntheticModule( @@ -77,15 +80,35 @@ export class EsmExecutor { return m } - public async createWebAssemblyModule(fileUrl: string, code: Buffer) { + public async createWebAssemblyModule(fileUrl: string, getCode: () => Buffer) { const cached = this.moduleCache.get(fileUrl) if (cached) return cached - const m = this.loadWebAssemblyModule(code, fileUrl) + const m = this.loadWebAssemblyModule(getCode(), fileUrl) this.moduleCache.set(fileUrl, m) return m } + public async createNetworkModule(fileUrl: string) { + // https://nodejs.org/api/esm.html#https-and-http-imports + if (fileUrl.startsWith('http:')) { + const url = new URL(fileUrl) + if ( + url.hostname !== 'localhost' + && url.hostname !== '::1' + && (IPnumber(url.hostname) & IPmask(8)) !== this.#httpIp + ) { + throw new Error( + // we don't know the importer, so it's undefined (the same happens in --pool=threads) + `import of '${fileUrl}' by undefined is not supported: ` + + 'http can only be used to load local resources (use https instead).', + ) + } + } + + return this.createEsModule(fileUrl, () => fetch(fileUrl).then(r => r.text())) + } + public async loadWebAssemblyModule(source: Buffer, identifier: string) { const cached = this.moduleCache.get(identifier) if (cached) @@ -187,6 +210,18 @@ export class EsmExecutor { return module } - return this.createEsModule(identifier, code) + return this.createEsModule(identifier, () => code) } } + +function IPnumber(address: string) { + const ip = address.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/) + if (ip) + return (+ip[1] << 24) + (+ip[2] << 16) + (+ip[3] << 8) + (+ip[4]) + + throw new Error(`Expected IP address, received ${address}`) +} + +function IPmask(maskSize: number) { + return -1 << (32 - maskSize) +} diff --git a/packages/vitest/src/runtime/vm/vite-executor.ts b/packages/vitest/src/runtime/vm/vite-executor.ts index 7a23f43f16b1..fe85810c1ad0 100644 --- a/packages/vitest/src/runtime/vm/vite-executor.ts +++ b/packages/vitest/src/runtime/vm/vite-executor.ts @@ -56,10 +56,12 @@ export class ViteExecutor { const cached = this.esm.resolveCachedModule(fileUrl) if (cached) return cached - const result = await this.options.transform(fileUrl, 'web') - if (!result.code) - throw new Error(`[vitest] Failed to transform ${fileUrl}. Does the file exist?`) - return this.esm.createEsModule(fileUrl, result.code) + return this.esm.createEsModule(fileUrl, async () => { + const result = await this.options.transform(fileUrl, 'web') + if (!result.code) + throw new Error(`[vitest] Failed to transform ${fileUrl}. Does the file exist?`) + return result.code + }) } private createViteClientModule() { diff --git a/test/cli/fixtures/network-imports/basic.test.ts b/test/cli/fixtures/network-imports/basic.test.ts index 21862089a4c9..0afdda86717d 100644 --- a/test/cli/fixtures/network-imports/basic.test.ts +++ b/test/cli/fixtures/network-imports/basic.test.ts @@ -9,3 +9,10 @@ import slash from 'http://localhost:9602/slash@3.0.0.js' test('network imports', () => { expect(slash('foo\\bar')).toBe('foo/bar') }) + +test('doesn\'t work for http outside localhost', async () => { + // @ts-expect-error network imports + await expect(() => import('http://100.0.0.0/')).rejects.toThrowError( + 'import of \'http://100.0.0.0/\' by undefined is not supported: http can only be used to load local resources (use https instead).', + ) +}) diff --git a/test/cli/test/network-imports.test.ts b/test/cli/test/network-imports.test.ts index 5e99e649e6a4..16209a2f6c06 100644 --- a/test/cli/test/network-imports.test.ts +++ b/test/cli/test/network-imports.test.ts @@ -9,11 +9,6 @@ const config = { forks: { execArgv: ['--experimental-network-imports'], }, - // not supported? - // FAIL test/basic.test.ts [ test/basic.test.ts ] - // Error: ENOENT: no such file or directory, open 'http://localhost:9602/slash@3.0.0.js' - // ❯ Object.openSync node:fs:596:3 - // ❯ readFileSync node:fs:464:35 vmThreads: { execArgv: ['--experimental-network-imports'], }, @@ -25,7 +20,7 @@ const config = { it.each([ 'threads', 'forks', - // 'vmThreads', + 'vmThreads', ])('importing from network in %s', async (pool) => { const { stderr, exitCode } = await runVitest({ ...config,