diff --git a/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs b/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs index e7fb092179315..24d45eb676647 100644 --- a/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs +++ b/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs @@ -111,23 +111,14 @@ export const command = { }); await Promise.all([ - time('extract relevant versions for packages', async () => { - log.info('extract relevant versions for packages'); - await moonRun(':extract-version-dependencies', { + time('prepare webpack bundles for packages', async () => { + log.info('extract relevant versions for packages and pre-build webpack bundles'); + await moonRun([':extract-version-dependencies', ':build-webpack'], { pipe: !quiet, quiet, noCache: forceInstall, }); - log.success('relevant versions extracted for packages'); - }), - time('pre-build webpack bundles for packages', async () => { - log.info('pre-build webpack bundles for packages'); - await moonRun(':build-webpack', { - pipe: !quiet, - quiet, - noCache: forceInstall, - }); - log.success('shared webpack bundles built'); + log.success('relevant versions extracted for packages and shared webpack bundles built'); }), shouldInstall ? time('run install scripts', async () => { diff --git a/src/dev/yarn/extract_version_dependencies.test.ts b/src/dev/yarn/extract_version_dependencies.test.ts new file mode 100644 index 0000000000000..6455e2a801290 --- /dev/null +++ b/src/dev/yarn/extract_version_dependencies.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { collectDependencyVersionLines } from './extract_version_dependencies'; + +describe('collectDependencyVersionLines', () => { + const rootPackageJsonContent = JSON.stringify({ + dependencies: { + alpha: '^1.0.0', + gamma: '^3.0.0', + }, + }); + + const yarnLockContent = ` +alpha@^1.0.0: + version "1.0.1" + dependencies: + beta "^1.0.0" + optional-child "^1.0.0" + +beta@^1.0.0: + version "1.1.0" + dependencies: + shared "^1.0.0" + +optional-child@^1.0.0: + version "1.0.2" + optionalDependencies: + shared "^2.0.0" + +gamma@^3.0.0: + version "3.0.0" + dependencies: + shared "^2.0.0" + +shared@^1.0.0: + version "1.2.0" + +shared@^2.0.0: + version "2.3.0" +`; + + it('returns direct resolved dependency versions when transitive is false', () => { + expect( + collectDependencyVersionLines({ + dependencies: ['alpha', 'gamma'], + rootPackageJsonContent, + transitive: false, + yarnLockContent, + }) + ).toEqual(['alpha@1.0.1', 'gamma@3.0.0']); + }); + + it('returns the full transitive closure including optional dependencies', () => { + expect( + collectDependencyVersionLines({ + dependencies: ['alpha'], + rootPackageJsonContent, + transitive: true, + yarnLockContent, + }) + ).toEqual([ + 'alpha@1.0.1', + 'beta@1.1.0', + 'optional-child@1.0.2', + 'shared@1.2.0', + 'shared@2.3.0', + ]); + }); + + it('keeps multiple resolved versions of the same package when they are both in the closure', () => { + expect( + collectDependencyVersionLines({ + dependencies: ['alpha', 'gamma'], + rootPackageJsonContent, + transitive: true, + yarnLockContent, + }) + ).toEqual([ + 'alpha@1.0.1', + 'beta@1.1.0', + 'gamma@3.0.0', + 'optional-child@1.0.2', + 'shared@1.2.0', + 'shared@2.3.0', + ]); + }); + + it('throws when a requested root dependency is not declared in package.json', () => { + expect(() => + collectDependencyVersionLines({ + dependencies: ['missing'], + rootPackageJsonContent, + transitive: true, + yarnLockContent, + }) + ).toThrowErrorMatchingInlineSnapshot( + `"Unable to find missing in the root package.json dependency list"` + ); + }); +}); diff --git a/src/dev/yarn/extract_version_dependencies.ts b/src/dev/yarn/extract_version_dependencies.ts index 389164b2877f5..33291f0e21b73 100644 --- a/src/dev/yarn/extract_version_dependencies.ts +++ b/src/dev/yarn/extract_version_dependencies.ts @@ -13,7 +13,8 @@ import fs from 'fs/promises'; import { run } from '@kbn/dev-cli-runner'; import { REPO_ROOT } from '@kbn/repo-info'; -import { parseYarnLockFile } from './yarn_lock_v1'; +import type { PackageInfo } from './yarn_lock_v1'; +import { parseYarnLock } from './yarn_lock_v1'; const options: RunOptions = { description: @@ -21,49 +22,149 @@ const options: RunOptions = { 'This can be useful to set up Moon task dependencies on package versions used in the repo.', flags: { string: ['collect'], + boolean: ['transitive'], }, - usage: `node scripts/extract_version_dependencies --collect `, + usage: + `node scripts/extract_version_dependencies ` + + `--collect [--collect ...] [--transitive]`, }; export async function runCli() { return run(async ({ flagsReader }) => { const outputFilePath = flagsReader.getPositionals()[0]; const dependencies = flagsReader.arrayOfStrings('collect'); + const transitive = flagsReader.boolean('transitive'); if (typeof dependencies === 'undefined') { throw new Error('--collect flag is required and must specify at least one package name.'); } - await collectDependenciesAndWriteFile(dependencies, outputFilePath); + await collectDependenciesAndWriteFile(dependencies, outputFilePath, { transitive }); return; }, options); } -async function collectDependenciesAndWriteFile(dependencies: string[], outputFilePath: string) { - const resolvedDependencyMap = new Map(); +async function collectDependenciesAndWriteFile( + dependencies: string[], + outputFilePath: string, + { transitive }: { transitive: boolean } +) { const rootPackageJson = path.join(REPO_ROOT, 'package.json'); const yarnLockPath = path.join(REPO_ROOT, 'yarn.lock'); - const pkgJson = await fs.readFile(rootPackageJson, 'utf-8').then((data) => JSON.parse(data)); - const yarnLockContent = parseYarnLockFile(yarnLockPath, dependencies); - const yarnLockEntries = Object.values(yarnLockContent); + const [packageJsonContent, yarnLockContent] = await Promise.all([ + fs.readFile(rootPackageJson, 'utf-8'), + fs.readFile(yarnLockPath, 'utf-8'), + ]); + + const outputLines = collectDependencyVersionLines({ + dependencies, + rootPackageJsonContent: packageJsonContent, + transitive, + yarnLockContent, + }); + + await fs.writeFile(outputFilePath, outputLines.join('\n') + '\n', 'utf-8'); +} + +export const collectDependencyVersionLines = ({ + dependencies, + rootPackageJsonContent, + transitive, + yarnLockContent, +}: { + dependencies: string[]; + rootPackageJsonContent: string; + transitive: boolean; + yarnLockContent: string; +}) => { + const pkgJson = JSON.parse(rootPackageJsonContent); + const yarnLockEntries = Object.values(parseYarnLock(yarnLockContent)); + const requestedVersionIndex = createRequestedVersionIndex(yarnLockEntries); const allRequestedDependencies = { ...pkgJson.devDependencies, ...pkgJson.dependencies, }; - dependencies.forEach((dep) => { - const declaredVersion = allRequestedDependencies[dep]; - const yarnlockEntry = yarnLockEntries.find((entry) => { - return entry.name === dep && entry.requestedVersions.includes(declaredVersion); - }); - resolvedDependencyMap.set(dep, yarnlockEntry!.resolvedVersion!); + const rootDependencies = resolveRootDependencies( + dependencies, + requestedVersionIndex, + allRequestedDependencies + ); + + const resolvedDependencyKeys = transitive + ? collectTransitiveDependencies(rootDependencies, requestedVersionIndex) + : new Set(rootDependencies.map((pkg) => `${pkg.name}@${pkg.resolvedVersion}`)); + + return Array.from(resolvedDependencyKeys).sort((a, b) => a.localeCompare(b)); +}; + +const createRequestedVersionIndex = (packages: PackageInfo[]) => { + const index = new Map(); + + for (const pkg of packages) { + for (const requestedVersion of pkg.requestedVersions) { + index.set(`${pkg.name}@${requestedVersion}`, pkg); + } + } + + return index; +}; + +const resolveDependency = ( + dependencyName: string, + requestedVersion: string, + requestedVersionIndex: Map +) => { + const pkg = requestedVersionIndex.get(`${dependencyName}@${requestedVersion}`); + + if (!pkg?.resolvedVersion) { + throw new Error( + `Unable to resolve ${dependencyName}@${requestedVersion} from yarn.lock dependency graph` + ); + } + + return pkg; +}; + +const resolveRootDependencies = ( + dependencies: string[], + requestedVersionIndex: Map, + allRequestedDependencies: Record +) => { + return dependencies.map((dependencyName) => { + const declaredVersion = allRequestedDependencies[dependencyName]; + + if (!declaredVersion) { + throw new Error(`Unable to find ${dependencyName} in the root package.json dependency list`); + } + + return resolveDependency(dependencyName, declaredVersion, requestedVersionIndex); }); +}; - const outputLines = Array.from(resolvedDependencyMap.entries()) - .sort((a, b) => a[0].localeCompare(b[0])) - .map(([name, version]) => `${name}@${version}`); +const collectTransitiveDependencies = ( + rootDependencies: PackageInfo[], + requestedVersionIndex: Map +) => { + const visited = new Set(); + const queue = [...rootDependencies]; - await fs.writeFile(outputFilePath, outputLines.join('\n') + '\n', 'utf-8'); -} + while (queue.length > 0) { + const pkg = queue.shift()!; + const packageKey = `${pkg.name}@${pkg.resolvedVersion}`; + + if (visited.has(packageKey)) { + continue; + } + + visited.add(packageKey); + + for (const [dependencyName, requestedVersion] of Object.entries(pkg.dependencies ?? {})) { + queue.push(resolveDependency(dependencyName, requestedVersion, requestedVersionIndex)); + } + } + + return visited; +}; diff --git a/src/dev/yarn/yarn_lock_v1.test.ts b/src/dev/yarn/yarn_lock_v1.test.ts new file mode 100644 index 0000000000000..837b6c7b8b13d --- /dev/null +++ b/src/dev/yarn/yarn_lock_v1.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { parseYarnLock } from './yarn_lock_v1'; + +describe('parseYarnLock', () => { + it('parses dependency versions that contain spaces and comparators', () => { + const lockfile = ` +stylis-plugin-rtl@>=2.1.0: + version "2.1.1" + dependencies: + cssjanus ">=1.3.2" + stylis "4.x" + +cssjanus@>=1.3.2: + version "2.3.0" + +stylis@4.x: + version "4.3.6" +`; + + expect(parseYarnLock(lockfile)).toMatchObject({ + 'stylis-plugin-rtl@2.1.1': { + dependencies: { + cssjanus: '>=1.3.2', + stylis: '4.x', + }, + }, + }); + }); + + it('includes optionalDependencies in the parsed dependency map', () => { + const lockfile = ` +wrapper@^1.0.0: + version "1.0.0" + optionalDependencies: + native-thing "^2.0.0" + +native-thing@^2.0.0: + version "2.1.0" +`; + + expect(parseYarnLock(lockfile)).toMatchObject({ + 'wrapper@1.0.0': { + dependencies: { + 'native-thing': '^2.0.0', + }, + }, + }); + }); + + it('merges duplicate lock entries for the same resolved package', () => { + const lockfile = ` +shared@^1.0.0: + version "1.2.3" + +shared@~1.2.0: + version "1.2.3" +`; + + expect(parseYarnLock(lockfile)).toEqual({ + 'shared@1.2.3': { + name: 'shared', + requestedVersions: ['^1.0.0', '~1.2.0'], + resolvedVersion: '1.2.3', + }, + }); + }); +}); diff --git a/src/dev/yarn/yarn_lock_v1.ts b/src/dev/yarn/yarn_lock_v1.ts index b49e321842864..bbb2d0d52bdcd 100644 --- a/src/dev/yarn/yarn_lock_v1.ts +++ b/src/dev/yarn/yarn_lock_v1.ts @@ -11,7 +11,7 @@ import fs from 'fs'; -interface PackageInfo { +export interface PackageInfo { name: string; requestedVersions: string[]; resolvedVersion?: string; @@ -22,6 +22,15 @@ interface PackageInfo { const makeKey = (name: string, version: string) => `${name}@${version}`; const trimQuotes = (str: string) => str.replace(/(^"|"$)/g, ''); +const splitDependencyLine = (line: string) => { + const match = line.trim().match(/^(\S+)\s+(.+)$/); + + if (!match) { + throw new Error(`Unable to parse yarn.lock dependency line: ${line}`); + } + + return [trimQuotes(match[1]), trimQuotes(match[2])] as const; +}; export const parseYarnLock = (content: string, focus?: string[]): Record => { const packages: Record = {}; @@ -57,20 +66,20 @@ export const parseYarnLock = (content: string, focus?: string[]): Record