diff --git a/.buildkite/scripts/bootstrap.sh b/.buildkite/scripts/bootstrap.sh index 6d5d793b73cec..ca5241c9d015b 100755 --- a/.buildkite/scripts/bootstrap.sh +++ b/.buildkite/scripts/bootstrap.sh @@ -29,10 +29,6 @@ if [[ "$(pwd)" != *"/local-ssd/"* && "$(pwd)" != "/dev/shm"* ]]; then mkdir -p ./.moon/cache echo "Extracting moon-cache.tar.gz to ./.moon/cache" tar -xzf ~/moon-cache.tar.gz -C ./ - elif [[ -d ~/.kibana-moon-cache ]]; then - echo "Using ~/.moon/cache as a starting point" - mkdir -p ./.moon/cache - mv ~/.kibana-moon-cache/* ./.moon/cache fi fi diff --git a/.buildkite/scripts/steps/store_cache.sh b/.buildkite/scripts/steps/store_cache.sh index 92ab546ef54af..66a0d41b28c87 100755 --- a/.buildkite/scripts/steps/store_cache.sh +++ b/.buildkite/scripts/steps/store_cache.sh @@ -6,6 +6,7 @@ source .buildkite/scripts/common/util.sh export MOON_CACHE=write .buildkite/scripts/bootstrap.sh + echo "--- Archive moon cache" if [[ ! -d .moon/cache ]]; then echo "No moon cache directory found, skipping archive" diff --git a/packages/kbn-plugin-helpers/src/tasks/build_webpack_packages.ts b/packages/kbn-plugin-helpers/src/tasks/build_webpack_packages.ts index 71d45468e8256..9877797d5c826 100644 --- a/packages/kbn-plugin-helpers/src/tasks/build_webpack_packages.ts +++ b/packages/kbn-plugin-helpers/src/tasks/build_webpack_packages.ts @@ -22,7 +22,10 @@ export async function buildWebpackPackages({ log, quiet, dist }: TaskContext) { const args = ['kbn', 'build-shared']; if (quiet) args.push('--quiet'); - if (dist) args.push('--dist'); + if (dist) { + args.push('--dist'); + args.push('--no-cache'); + } await execa('yarn', args, { cwd: REPO_ROOT, stdio }); } diff --git a/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs b/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs index 1589eddbdad37..de47d1344ec8f 100644 --- a/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs +++ b/src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs @@ -111,23 +111,14 @@ export const command = { }); await Promise.all([ - time('extract relevant versions for packages', async () => { - log.info('extract relevant versions for packages'); - await moonRun(':extract-version-dependencies', { + time('prepare webpack bundles for packages', async () => { + log.info('pre-build webpack bundles'); + await moonRun([':build-webpack'], { pipe: !quiet, quiet, noCache: forceInstall, }); - log.success('relevant versions extracted for packages'); - }), - time('pre-build webpack bundles for packages', async () => { - log.info('pre-build webpack bundles for packages'); - await moonRun(':build-webpack', { - pipe: !quiet, - quiet, - noCache: forceInstall, - }); - log.success('shared webpack bundles built'); + log.success('relevant versions extracted for packages and shared webpack bundles built'); }), time('run install scripts', async () => { await runInstallScripts(log, { quiet }); diff --git a/src/dev/yarn/extract_version_dependencies.test.ts b/src/dev/yarn/extract_version_dependencies.test.ts new file mode 100644 index 0000000000000..6455e2a801290 --- /dev/null +++ b/src/dev/yarn/extract_version_dependencies.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { collectDependencyVersionLines } from './extract_version_dependencies'; + +describe('collectDependencyVersionLines', () => { + const rootPackageJsonContent = JSON.stringify({ + dependencies: { + alpha: '^1.0.0', + gamma: '^3.0.0', + }, + }); + + const yarnLockContent = ` +alpha@^1.0.0: + version "1.0.1" + dependencies: + beta "^1.0.0" + optional-child "^1.0.0" + +beta@^1.0.0: + version "1.1.0" + dependencies: + shared "^1.0.0" + +optional-child@^1.0.0: + version "1.0.2" + optionalDependencies: + shared "^2.0.0" + +gamma@^3.0.0: + version "3.0.0" + dependencies: + shared "^2.0.0" + +shared@^1.0.0: + version "1.2.0" + +shared@^2.0.0: + version "2.3.0" +`; + + it('returns direct resolved dependency versions when transitive is false', () => { + expect( + collectDependencyVersionLines({ + dependencies: ['alpha', 'gamma'], + rootPackageJsonContent, + transitive: false, + yarnLockContent, + }) + ).toEqual(['alpha@1.0.1', 'gamma@3.0.0']); + }); + + it('returns the full transitive closure including optional dependencies', () => { + expect( + collectDependencyVersionLines({ + dependencies: ['alpha'], + rootPackageJsonContent, + transitive: true, + yarnLockContent, + }) + ).toEqual([ + 'alpha@1.0.1', + 'beta@1.1.0', + 'optional-child@1.0.2', + 'shared@1.2.0', + 'shared@2.3.0', + ]); + }); + + it('keeps multiple resolved versions of the same package when they are both in the closure', () => { + expect( + collectDependencyVersionLines({ + dependencies: ['alpha', 'gamma'], + rootPackageJsonContent, + transitive: true, + yarnLockContent, + }) + ).toEqual([ + 'alpha@1.0.1', + 'beta@1.1.0', + 'gamma@3.0.0', + 'optional-child@1.0.2', + 'shared@1.2.0', + 'shared@2.3.0', + ]); + }); + + it('throws when a requested root dependency is not declared in package.json', () => { + expect(() => + collectDependencyVersionLines({ + dependencies: ['missing'], + rootPackageJsonContent, + transitive: true, + yarnLockContent, + }) + ).toThrowErrorMatchingInlineSnapshot( + `"Unable to find missing in the root package.json dependency list"` + ); + }); +}); diff --git a/src/dev/yarn/extract_version_dependencies.ts b/src/dev/yarn/extract_version_dependencies.ts index 389164b2877f5..33291f0e21b73 100644 --- a/src/dev/yarn/extract_version_dependencies.ts +++ b/src/dev/yarn/extract_version_dependencies.ts @@ -13,7 +13,8 @@ import fs from 'fs/promises'; import { run } from '@kbn/dev-cli-runner'; import { REPO_ROOT } from '@kbn/repo-info'; -import { parseYarnLockFile } from './yarn_lock_v1'; +import type { PackageInfo } from './yarn_lock_v1'; +import { parseYarnLock } from './yarn_lock_v1'; const options: RunOptions = { description: @@ -21,49 +22,149 @@ const options: RunOptions = { 'This can be useful to set up Moon task dependencies on package versions used in the repo.', flags: { string: ['collect'], + boolean: ['transitive'], }, - usage: `node scripts/extract_version_dependencies --collect `, + usage: + `node scripts/extract_version_dependencies ` + + `--collect [--collect ...] [--transitive]`, }; export async function runCli() { return run(async ({ flagsReader }) => { const outputFilePath = flagsReader.getPositionals()[0]; const dependencies = flagsReader.arrayOfStrings('collect'); + const transitive = flagsReader.boolean('transitive'); if (typeof dependencies === 'undefined') { throw new Error('--collect flag is required and must specify at least one package name.'); } - await collectDependenciesAndWriteFile(dependencies, outputFilePath); + await collectDependenciesAndWriteFile(dependencies, outputFilePath, { transitive }); return; }, options); } -async function collectDependenciesAndWriteFile(dependencies: string[], outputFilePath: string) { - const resolvedDependencyMap = new Map(); +async function collectDependenciesAndWriteFile( + dependencies: string[], + outputFilePath: string, + { transitive }: { transitive: boolean } +) { const rootPackageJson = path.join(REPO_ROOT, 'package.json'); const yarnLockPath = path.join(REPO_ROOT, 'yarn.lock'); - const pkgJson = await fs.readFile(rootPackageJson, 'utf-8').then((data) => JSON.parse(data)); - const yarnLockContent = parseYarnLockFile(yarnLockPath, dependencies); - const yarnLockEntries = Object.values(yarnLockContent); + const [packageJsonContent, yarnLockContent] = await Promise.all([ + fs.readFile(rootPackageJson, 'utf-8'), + fs.readFile(yarnLockPath, 'utf-8'), + ]); + + const outputLines = collectDependencyVersionLines({ + dependencies, + rootPackageJsonContent: packageJsonContent, + transitive, + yarnLockContent, + }); + + await fs.writeFile(outputFilePath, outputLines.join('\n') + '\n', 'utf-8'); +} + +export const collectDependencyVersionLines = ({ + dependencies, + rootPackageJsonContent, + transitive, + yarnLockContent, +}: { + dependencies: string[]; + rootPackageJsonContent: string; + transitive: boolean; + yarnLockContent: string; +}) => { + const pkgJson = JSON.parse(rootPackageJsonContent); + const yarnLockEntries = Object.values(parseYarnLock(yarnLockContent)); + const requestedVersionIndex = createRequestedVersionIndex(yarnLockEntries); const allRequestedDependencies = { ...pkgJson.devDependencies, ...pkgJson.dependencies, }; - dependencies.forEach((dep) => { - const declaredVersion = allRequestedDependencies[dep]; - const yarnlockEntry = yarnLockEntries.find((entry) => { - return entry.name === dep && entry.requestedVersions.includes(declaredVersion); - }); - resolvedDependencyMap.set(dep, yarnlockEntry!.resolvedVersion!); + const rootDependencies = resolveRootDependencies( + dependencies, + requestedVersionIndex, + allRequestedDependencies + ); + + const resolvedDependencyKeys = transitive + ? collectTransitiveDependencies(rootDependencies, requestedVersionIndex) + : new Set(rootDependencies.map((pkg) => `${pkg.name}@${pkg.resolvedVersion}`)); + + return Array.from(resolvedDependencyKeys).sort((a, b) => a.localeCompare(b)); +}; + +const createRequestedVersionIndex = (packages: PackageInfo[]) => { + const index = new Map(); + + for (const pkg of packages) { + for (const requestedVersion of pkg.requestedVersions) { + index.set(`${pkg.name}@${requestedVersion}`, pkg); + } + } + + return index; +}; + +const resolveDependency = ( + dependencyName: string, + requestedVersion: string, + requestedVersionIndex: Map +) => { + const pkg = requestedVersionIndex.get(`${dependencyName}@${requestedVersion}`); + + if (!pkg?.resolvedVersion) { + throw new Error( + `Unable to resolve ${dependencyName}@${requestedVersion} from yarn.lock dependency graph` + ); + } + + return pkg; +}; + +const resolveRootDependencies = ( + dependencies: string[], + requestedVersionIndex: Map, + allRequestedDependencies: Record +) => { + return dependencies.map((dependencyName) => { + const declaredVersion = allRequestedDependencies[dependencyName]; + + if (!declaredVersion) { + throw new Error(`Unable to find ${dependencyName} in the root package.json dependency list`); + } + + return resolveDependency(dependencyName, declaredVersion, requestedVersionIndex); }); +}; - const outputLines = Array.from(resolvedDependencyMap.entries()) - .sort((a, b) => a[0].localeCompare(b[0])) - .map(([name, version]) => `${name}@${version}`); +const collectTransitiveDependencies = ( + rootDependencies: PackageInfo[], + requestedVersionIndex: Map +) => { + const visited = new Set(); + const queue = [...rootDependencies]; - await fs.writeFile(outputFilePath, outputLines.join('\n') + '\n', 'utf-8'); -} + while (queue.length > 0) { + const pkg = queue.shift()!; + const packageKey = `${pkg.name}@${pkg.resolvedVersion}`; + + if (visited.has(packageKey)) { + continue; + } + + visited.add(packageKey); + + for (const [dependencyName, requestedVersion] of Object.entries(pkg.dependencies ?? {})) { + queue.push(resolveDependency(dependencyName, requestedVersion, requestedVersionIndex)); + } + } + + return visited; +}; diff --git a/src/dev/yarn/yarn_lock_v1.test.ts b/src/dev/yarn/yarn_lock_v1.test.ts new file mode 100644 index 0000000000000..03fe0b3424f8a --- /dev/null +++ b/src/dev/yarn/yarn_lock_v1.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { parseYarnLock } from './yarn_lock_v1'; + +describe('parseYarnLock', () => { + it('parses dependency versions that contain spaces and comparators', () => { + const lockfile = ` +stylis-plugin-rtl@>=2.1.0: + version "2.1.1" + dependencies: + cssjanus ">=1.3.2" + stylis "4.x" + +cssjanus@>=1.3.2: + version "2.3.0" + +stylis@4.x: + version "4.3.6" +`; + + expect(parseYarnLock(lockfile)).toMatchObject({ + 'stylis-plugin-rtl@2.1.1': { + dependencies: { + cssjanus: '>=1.3.2', + stylis: '4.x', + }, + }, + }); + }); + + it('includes optionalDependencies in the parsed dependency map', () => { + const lockfile = ` +wrapper@^1.0.0: + version "1.0.0" + optionalDependencies: + native-thing "^2.0.0" + +native-thing@^2.0.0: + version "2.1.0" +`; + + expect(parseYarnLock(lockfile)).toMatchObject({ + 'wrapper@1.0.0': { + dependencies: { + 'native-thing': '^2.0.0', + }, + }, + }); + }); + + it('merges duplicate lock entries for the same resolved package', () => { + const lockfile = ` +shared@^1.0.0: + version "1.2.3" + +shared@~1.2.0: + version "1.2.3" +`; + + expect(parseYarnLock(lockfile)).toEqual({ + 'shared@1.2.3': { + name: 'shared', + requestedVersions: ['^1.0.0', '~1.2.0'], + resolvedVersion: '1.2.3', + }, + }); + }); + + it('indexes each package alias in a compound yarn.lock header', () => { + const lockfile = ` +"@scope/alias@npm:@scope/alias@2.0.1", "plain-name@1 - 2", "plain-name@npm:@scope/alias@2.0.1": + version "2.0.1" + resolved "https://example.invalid/pkg.tgz" +`; + + const parsed = parseYarnLock(lockfile); + + expect(parsed['plain-name@2.0.1']).toMatchObject({ + name: 'plain-name', + requestedVersions: ['1 - 2', 'npm:@scope/alias@2.0.1'], + resolvedVersion: '2.0.1', + }); + expect(parsed['@scope/alias@2.0.1']).toMatchObject({ + name: '@scope/alias', + requestedVersions: ['npm:@scope/alias@2.0.1'], + resolvedVersion: '2.0.1', + }); + }); +}); diff --git a/src/dev/yarn/yarn_lock_v1.ts b/src/dev/yarn/yarn_lock_v1.ts index b49e321842864..0ae6dd43be5d1 100644 --- a/src/dev/yarn/yarn_lock_v1.ts +++ b/src/dev/yarn/yarn_lock_v1.ts @@ -11,7 +11,7 @@ import fs from 'fs'; -interface PackageInfo { +export interface PackageInfo { name: string; requestedVersions: string[]; resolvedVersion?: string; @@ -23,6 +23,32 @@ interface PackageInfo { const makeKey = (name: string, version: string) => `${name}@${version}`; const trimQuotes = (str: string) => str.replace(/(^"|"$)/g, ''); +/** Splits a yarn.lock header descriptor into [packageName, requestedRangeOrDescriptor]. */ +const splitYarnLockDescriptor = (entry: string): [string, string] => { + if (entry.startsWith('@')) { + const versionSeparator = entry.indexOf('@', 1); + if (versionSeparator === -1) { + throw new Error(`Invalid yarn.lock descriptor: ${entry}`); + } + return [entry.slice(0, versionSeparator), entry.slice(versionSeparator + 1)]; + } + const at = entry.indexOf('@'); + if (at === -1) { + throw new Error(`Invalid yarn.lock descriptor: ${entry}`); + } + return [entry.slice(0, at), entry.slice(at + 1)]; +}; + +const splitDependencyLine = (line: string) => { + const match = line.trim().match(/^(\S+)\s+(.+)$/); + + if (!match) { + throw new Error(`Unable to parse yarn.lock dependency line: ${line}`); + } + + return [trimQuotes(match[1]), trimQuotes(match[2])] as const; +}; + export const parseYarnLock = (content: string, focus?: string[]): Record => { const packages: Record = {}; const contentWithoutComments = content.replace(/#.*$/gm, '').trim(); @@ -35,42 +61,45 @@ export const parseYarnLock = (content: string, focus?: string[]): Record { + const [pkgName] = splitYarnLockDescriptor(entry); + return focusSet.has(pkgName); + }); + if (!anyFocused) { + continue; + } } - const requestedVersions = headerEntries.map((entry) => entry.substring(name.length + 1)); - - const packageInfo: PackageInfo = { - name, - requestedVersions, - }; + let resolvedVersion: string | undefined; + let resolvedUrl: string | undefined; + let integrity: string | undefined; + let blockDependencies: { [key: string]: string } | undefined; for (let i = 1; i < lines.length; i++) { const line = lines[i].trim(); const [key, value] = line.split(/\s+/, 2).map(trimQuotes); if (key === 'version') { - packageInfo.resolvedVersion = value; + resolvedVersion = value; } else if (key === 'resolved') { - packageInfo.resolvedUrl = value; + resolvedUrl = value; } else if (key === 'integrity') { - packageInfo.integrity = value; - } else if (key === 'dependencies:') { + integrity = value; + } else if (key === 'dependencies:' || key === 'optionalDependencies:') { let depCount = 0; if (focusSet === null) { - packageInfo.dependencies = {}; + blockDependencies = blockDependencies || {}; for (let j = i + 1; j < lines.length; j++) { const depLine = lines[j]; - if (depLine.trim() === '') break; - const [depKey, depVersion] = depLine.trim().split(/\s+/).map(trimQuotes); - packageInfo.dependencies![depKey] = depVersion; + if (!/^\s{4,}\S/.test(depLine)) break; + const [depKey, depVersion] = splitDependencyLine(depLine); + blockDependencies[depKey] = depVersion; depCount++; } } else { for (let j = i + 1; j < lines.length; j++) { - if (lines[j].trim() === '') break; + if (!/^\s{4,}\S/.test(lines[j])) break; depCount++; } } @@ -78,15 +107,35 @@ export const parseYarnLock = (content: string, focus?: string[]): Record