Reapply "Fix Moon webpack dependency cache invalidation (#258940)"

src/dev/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs

@@ -111,23 +111,14 @@ export const command = {
     });
 
     await Promise.all([
-      time('extract relevant versions for packages', async () => {
-        log.info('extract relevant versions for packages');
-        await moonRun(':extract-version-dependencies', {
+      time('prepare webpack bundles for packages', async () => {
+        log.info('extract relevant versions for packages and pre-build webpack bundles');
+        await moonRun([':extract-version-dependencies', ':build-webpack'], {
           pipe: !quiet,
           quiet,
           noCache: forceInstall,
         });
-        log.success('relevant versions extracted for packages');
-      }),
-      time('pre-build webpack bundles for packages', async () => {
-        log.info('pre-build webpack bundles for packages');
-        await moonRun(':build-webpack', {
-          pipe: !quiet,
-          quiet,
-          noCache: forceInstall,
-        });
-        log.success('shared webpack bundles built');
+        log.success('relevant versions extracted for packages and shared webpack bundles built');
       }),
       shouldInstall
         ? time('run install scripts', async () => {

src/dev/yarn/extract_version_dependencies.test.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { collectDependencyVersionLines } from './extract_version_dependencies';
+
+describe('collectDependencyVersionLines', () => {
+  const rootPackageJsonContent = JSON.stringify({
+    dependencies: {
+      alpha: '^1.0.0',
+      gamma: '^3.0.0',
+    },
+  });
+
+  const yarnLockContent = `
+alpha@^1.0.0:
+  version "1.0.1"
+  dependencies:
+    beta "^1.0.0"
+    optional-child "^1.0.0"
+
+beta@^1.0.0:
+  version "1.1.0"
+  dependencies:
+    shared "^1.0.0"
+
+optional-child@^1.0.0:
+  version "1.0.2"
+  optionalDependencies:
+    shared "^2.0.0"
+
+gamma@^3.0.0:
+  version "3.0.0"
+  dependencies:
+    shared "^2.0.0"
+
+shared@^1.0.0:
+  version "1.2.0"
+
+shared@^2.0.0:
+  version "2.3.0"
+`;
+
+  it('returns direct resolved dependency versions when transitive is false', () => {
+    expect(
+      collectDependencyVersionLines({
+        dependencies: ['alpha', 'gamma'],
+        rootPackageJsonContent,
+        transitive: false,
+        yarnLockContent,
+      })
+    ).toEqual(['alpha@1.0.1', 'gamma@3.0.0']);
+  });
+
+  it('returns the full transitive closure including optional dependencies', () => {
+    expect(
+      collectDependencyVersionLines({
+        dependencies: ['alpha'],
+        rootPackageJsonContent,
+        transitive: true,
+        yarnLockContent,
+      })
+    ).toEqual([
+      'alpha@1.0.1',
+      'beta@1.1.0',
+      'optional-child@1.0.2',
+      'shared@1.2.0',
+      'shared@2.3.0',
+    ]);
+  });
+
+  it('keeps multiple resolved versions of the same package when they are both in the closure', () => {
+    expect(
+      collectDependencyVersionLines({
+        dependencies: ['alpha', 'gamma'],
+        rootPackageJsonContent,
+        transitive: true,
+        yarnLockContent,
+      })
+    ).toEqual([
+      'alpha@1.0.1',
+      'beta@1.1.0',
+      'gamma@3.0.0',
+      'optional-child@1.0.2',
+      'shared@1.2.0',
+      'shared@2.3.0',
+    ]);
+  });
+
+  it('throws when a requested root dependency is not declared in package.json', () => {
+    expect(() =>
+      collectDependencyVersionLines({
+        dependencies: ['missing'],
+        rootPackageJsonContent,
+        transitive: true,
+        yarnLockContent,
+      })
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"Unable to find missing in the root package.json dependency list"`
+    );
+  });
+});

src/dev/yarn/extract_version_dependencies.ts

@@ -13,57 +13,158 @@ import fs from 'fs/promises';
 
 import { run } from '@kbn/dev-cli-runner';
 import { REPO_ROOT } from '@kbn/repo-info';
-import { parseYarnLockFile } from './yarn_lock_v1';
+import type { PackageInfo } from './yarn_lock_v1';
+import { parseYarnLock } from './yarn_lock_v1';
 
 const options: RunOptions = {
   description:
     'Extracts declared dependency versions from a Yarn v1 lockfile into a versions file.\n' +
     'This can be useful to set up Moon task dependencies on package versions used in the repo.',
   flags: {
     string: ['collect'],
+    boolean: ['transitive'],
   },
-  usage: `node scripts/extract_version_dependencies <output_file_path> --collect <package_name1,package_name2,...>`,
+  usage:
+    `node scripts/extract_version_dependencies <output_file_path> ` +
+    `--collect <package_name1> [--collect <package_name2> ...] [--transitive]`,
 };
 
 export async function runCli() {
   return run(async ({ flagsReader }) => {
     const outputFilePath = flagsReader.getPositionals()[0];
     const dependencies = flagsReader.arrayOfStrings('collect');
+    const transitive = flagsReader.boolean('transitive');
     if (typeof dependencies === 'undefined') {
       throw new Error('--collect flag is required and must specify at least one package name.');
     }
 
-    await collectDependenciesAndWriteFile(dependencies, outputFilePath);
+    await collectDependenciesAndWriteFile(dependencies, outputFilePath, { transitive });
 
     return;
   }, options);
 }
 
-async function collectDependenciesAndWriteFile(dependencies: string[], outputFilePath: string) {
-  const resolvedDependencyMap = new Map<string, string>();
+async function collectDependenciesAndWriteFile(
+  dependencies: string[],
+  outputFilePath: string,
+  { transitive }: { transitive: boolean }
+) {
   const rootPackageJson = path.join(REPO_ROOT, 'package.json');
   const yarnLockPath = path.join(REPO_ROOT, 'yarn.lock');
 
-  const pkgJson = await fs.readFile(rootPackageJson, 'utf-8').then((data) => JSON.parse(data));
-  const yarnLockContent = parseYarnLockFile(yarnLockPath, dependencies);
-  const yarnLockEntries = Object.values(yarnLockContent);
+  const [packageJsonContent, yarnLockContent] = await Promise.all([
+    fs.readFile(rootPackageJson, 'utf-8'),
+    fs.readFile(yarnLockPath, 'utf-8'),
+  ]);
+
+  const outputLines = collectDependencyVersionLines({
+    dependencies,
+    rootPackageJsonContent: packageJsonContent,
+    transitive,
+    yarnLockContent,
+  });
+
+  await fs.writeFile(outputFilePath, outputLines.join('\n') + '\n', 'utf-8');
+}
+
+export const collectDependencyVersionLines = ({
+  dependencies,
+  rootPackageJsonContent,
+  transitive,
+  yarnLockContent,
+}: {
+  dependencies: string[];
+  rootPackageJsonContent: string;
+  transitive: boolean;
+  yarnLockContent: string;
+}) => {
+  const pkgJson = JSON.parse(rootPackageJsonContent);
+  const yarnLockEntries = Object.values(parseYarnLock(yarnLockContent));
+  const requestedVersionIndex = createRequestedVersionIndex(yarnLockEntries);
 
   const allRequestedDependencies = {
     ...pkgJson.devDependencies,
     ...pkgJson.dependencies,
   };
 
-  dependencies.forEach((dep) => {
-    const declaredVersion = allRequestedDependencies[dep];
-    const yarnlockEntry = yarnLockEntries.find((entry) => {
-      return entry.name === dep && entry.requestedVersions.includes(declaredVersion);
-    });
-    resolvedDependencyMap.set(dep, yarnlockEntry!.resolvedVersion!);
+  const rootDependencies = resolveRootDependencies(
+    dependencies,
+    requestedVersionIndex,
+    allRequestedDependencies
+  );
+
+  const resolvedDependencyKeys = transitive
+    ? collectTransitiveDependencies(rootDependencies, requestedVersionIndex)
+    : new Set(rootDependencies.map((pkg) => `${pkg.name}@${pkg.resolvedVersion}`));
+
+  return Array.from(resolvedDependencyKeys).sort((a, b) => a.localeCompare(b));
+};
+
+const createRequestedVersionIndex = (packages: PackageInfo[]) => {
+  const index = new Map<string, PackageInfo>();
+
+  for (const pkg of packages) {
+    for (const requestedVersion of pkg.requestedVersions) {
+      index.set(`${pkg.name}@${requestedVersion}`, pkg);
+    }
+  }
+
+  return index;
+};
+
+const resolveDependency = (
+  dependencyName: string,
+  requestedVersion: string,
+  requestedVersionIndex: Map<string, PackageInfo>
+) => {
+  const pkg = requestedVersionIndex.get(`${dependencyName}@${requestedVersion}`);
+
+  if (!pkg?.resolvedVersion) {
+    throw new Error(
+      `Unable to resolve ${dependencyName}@${requestedVersion} from yarn.lock dependency graph`
+    );
+  }
+
+  return pkg;
+};
+
+const resolveRootDependencies = (
+  dependencies: string[],
+  requestedVersionIndex: Map<string, PackageInfo>,
+  allRequestedDependencies: Record<string, string>
+) => {
+  return dependencies.map((dependencyName) => {
+    const declaredVersion = allRequestedDependencies[dependencyName];
+
+    if (!declaredVersion) {
+      throw new Error(`Unable to find ${dependencyName} in the root package.json dependency list`);
+    }
+
+    return resolveDependency(dependencyName, declaredVersion, requestedVersionIndex);
   });
+};
 
-  const outputLines = Array.from(resolvedDependencyMap.entries())
-    .sort((a, b) => a[0].localeCompare(b[0]))
-    .map(([name, version]) => `${name}@${version}`);
+const collectTransitiveDependencies = (
+  rootDependencies: PackageInfo[],
+  requestedVersionIndex: Map<string, PackageInfo>
+) => {
+  const visited = new Set<string>();
+  const queue = [...rootDependencies];
 
-  await fs.writeFile(outputFilePath, outputLines.join('\n') + '\n', 'utf-8');
-}
+  while (queue.length > 0) {
+    const pkg = queue.shift()!;
+    const packageKey = `${pkg.name}@${pkg.resolvedVersion}`;
+
+    if (visited.has(packageKey)) {
+      continue;
+    }
+
+    visited.add(packageKey);
+
+    for (const [dependencyName, requestedVersion] of Object.entries(pkg.dependencies ?? {})) {
+      queue.push(resolveDependency(dependencyName, requestedVersion, requestedVersionIndex));
+    }
+  }
+
+  return visited;
+};

src/dev/yarn/yarn_lock_v1.test.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { parseYarnLock } from './yarn_lock_v1';
+
+describe('parseYarnLock', () => {
+  it('parses dependency versions that contain spaces and comparators', () => {
+    const lockfile = `
+stylis-plugin-rtl@>=2.1.0:
+  version "2.1.1"
+  dependencies:
+    cssjanus ">=1.3.2"
+    stylis "4.x"
+
+cssjanus@>=1.3.2:
+  version "2.3.0"
+
+stylis@4.x:
+  version "4.3.6"
+`;
+
+    expect(parseYarnLock(lockfile)).toMatchObject({
+      'stylis-plugin-rtl@2.1.1': {
+        dependencies: {
+          cssjanus: '>=1.3.2',
+          stylis: '4.x',
+        },
+      },
+    });
+  });
+
+  it('includes optionalDependencies in the parsed dependency map', () => {
+    const lockfile = `
+wrapper@^1.0.0:
+  version "1.0.0"
+  optionalDependencies:
+    native-thing "^2.0.0"
+
+native-thing@^2.0.0:
+  version "2.1.0"
+`;
+
+    expect(parseYarnLock(lockfile)).toMatchObject({
+      'wrapper@1.0.0': {
+        dependencies: {
+          'native-thing': '^2.0.0',
+        },
+      },
+    });
+  });
+
+  it('merges duplicate lock entries for the same resolved package', () => {
+    const lockfile = `
+shared@^1.0.0:
+  version "1.2.3"
+
+shared@~1.2.0:
+  version "1.2.3"
+`;
+
+    expect(parseYarnLock(lockfile)).toEqual({
+      'shared@1.2.3': {
+        name: 'shared',
+        requestedVersions: ['^1.0.0', '~1.2.0'],
+        resolvedVersion: '1.2.3',
+      },
+    });
+  });
+});