[9.3] chore(NA): improve performance of kibana dev/build distributable (#256377)

.buildkite/scripts/steps/security/third_party_packages.txt

@@ -25,3 +25,7 @@ ink
 @hey-api/openapi-ts
 vscode-languageserver-textdocument
 yaml-language-server
+@swc/core
+lightningcss
+tar-fs
+browserslist

package.json

@@ -1771,6 +1771,7 @@
     "@storybook/test": "8.6.17",
     "@storybook/theming": "8.6.17",
     "@storybook/types": "8.6.17",
+    "@swc/core": "1.15.18",
     "@testing-library/dom": "10.4.1",
     "@testing-library/jest-dom": "6.9.1",
     "@testing-library/react": "16.3.2",
@@ -1914,6 +1915,7 @@
     "babel-plugin-transform-typescript-metadata": "0.3.2",
     "backport": "10.2.0",
     "blob-polyfill": "9.0.20240710",
+    "browserslist": "4.28.0",
     "buildkite-test-collector": "1.8.1",
     "cache-manager": "7.0.0",
     "cache-manager-fs-hash": "2.0.0",
@@ -2000,6 +2002,7 @@
     "keyv": "5.3.4",
     "license-checker": "25.0.1",
     "lighthouse": "12.8.2",
+    "lightningcss": "1.31.1",
     "listr2": "8.2.5",
     "lmdb": "3.4.4",
     "marge": "1.0.1",
@@ -2058,6 +2061,7 @@
     "swagger-ui-express": "5.0.1",
     "table": "6.9.0",
     "tape": "5.9.0",
+    "tar-fs": "3.1.1",
     "terser": "5.40.0",
     "terser-webpack-plugin": "5.3.17",
     "tough-cookie": "6.0.0",

packages/kbn-yarn-install-scripts/config.json

@@ -113,6 +113,12 @@
       "lifecycle": "install",
       "required": false,
       "reason": "Builds from source if npm_config_build_from_source=true.  Prebuilds are sufficient."
+    },
+    {
+      "path": "@swc/core",
+      "lifecycle": "postinstall",
+      "required": false,
+      "reason": "Builds native for @swc/core, used by proxy at src/dev/build to optimize assets.  Prebuilds are sufficient."
     }
   ]
-}
+}

renovate.json

@@ -4868,6 +4868,42 @@
       ],
       "enabled": true,
       "minimumReleaseAge": "14 days"
+    },
+    {
+      "groupName": "@swc/core",
+      "matchDepNames": ["@swc/core"],
+      "reviewers": ["team:kibana-operations"],
+      "matchBaseBranches": ["main"],
+      "labels": ["Team:Operations", "release_note:skip"],
+      "minimumReleaseAge": "14 days",
+      "enabled": true
+    },
+    {
+      "groupName": "lightningcss",
+      "matchDepNames": ["lightningcss"],
+      "reviewers": ["team:kibana-operations"],
+      "matchBaseBranches": ["main"],
+      "labels": ["Team:Operations", "release_note:skip"],
+      "minimumReleaseAge": "14 days",
+      "enabled": true
+    },
+    {
+      "groupName": "tar-fs",
+      "matchDepNames": ["tar-fs"],
+      "reviewers": ["team:kibana-operations"],
+      "matchBaseBranches": ["main"],
+      "labels": ["Team:Operations", "release_note:skip"],
+      "minimumReleaseAge": "14 days",
+      "enabled": true
+    },
+    {
+      "groupName": "browserslist",
+      "matchDepNames": ["browserslist"],
+      "reviewers": ["team:kibana-operations"],
+      "matchBaseBranches": ["main"],
+      "labels": ["Team:Operations", "release_note:skip"],
+      "minimumReleaseAge": "14 days",
+      "enabled": true
     }
   ],
   "customManagers": [
@@ -4906,4 +4942,4 @@
       "datasourceTemplate": "docker"
     }
   ]
-}
+}

src/dev/build/build_distributables.ts

@@ -88,10 +88,12 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
     await globalRun(Tasks.CreateXPackNoticeFile);
 
     await globalRun(Tasks.DeletePackagesFromBuildRoot);
+
     await globalRun(Tasks.UpdateLicenseFile);
     await globalRun(Tasks.RemovePackageJsonDeps);
     await globalRun(Tasks.CleanPackageManagerRelatedFiles);
     await globalRun(Tasks.CleanExtraFilesFromModules);
+
     await globalRun(Tasks.CleanEmptyFolders);
     await globalRun(Tasks.FetchAgentVersionsList);
   }

src/dev/build/lib/fs.ts

@@ -12,15 +12,16 @@ import Fsp from 'fs/promises';
 import * as Rx from 'rxjs';
 import { createHash } from 'crypto';
 import { pipeline } from 'stream/promises';
-import { resolve, dirname, isAbsolute, sep } from 'path';
-import { createGunzip } from 'zlib';
+import { resolve, dirname, isAbsolute, basename, sep } from 'path';
+import { createGunzip, createGzip } from 'zlib';
 import { inspect } from 'util';
 
 import archiver from 'archiver';
 import globby from 'globby';
 import cpy from 'cpy';
 import del from 'del';
 import * as tar from 'tar';
+import { pack as tarFsPack } from 'tar-fs';
 import type { ToolingLog } from '@kbn/tooling-log';
 
 export function assertAbsolute(path: string) {
@@ -231,29 +232,37 @@ interface CompressTarOptions {
   rootDirectoryName?: string;
   source: string;
   destination: string;
-  archiverOptions?: archiver.TarOptions & archiver.CoreOptions;
+  gzipLevel?: number;
 }
 export async function compressTar({
   source,
   destination,
-  archiverOptions,
+  gzipLevel = 9,
   createRootDirectory,
   rootDirectoryName,
-}: CompressTarOptions) {
-  const output = fs.createWriteStream(destination);
-  const archive = archiver('tar', archiverOptions);
-  const folder = rootDirectoryName ? rootDirectoryName : source.split(sep).slice(-1)[0];
-  const name = createRootDirectory ? folder : false;
-  archive.pipe(output);
+}: CompressTarOptions): Promise<number> {
+  assertAbsolute(source);
+  assertAbsolute(destination);
+
+  const folder = rootDirectoryName || basename(source);
 
   let fileCount = 0;
-  archive.on('entry', (entry) => {
-    if (entry.stats?.isFile()) {
-      fileCount += 1;
-    }
+  const packStream = tarFsPack(source, {
+    map(header) {
+      if (header.type === 'file') {
+        fileCount += 1;
+      }
+      if (createRootDirectory) {
+        header.name = folder + '/' + header.name;
+      }
+      return header;
+    },
   });
 
-  await archive.directory(source, name).finalize();
+  const gzip = createGzip({ level: gzipLevel });
+  const output = fs.createWriteStream(destination);
+
+  await pipeline(packStream, gzip, output);
 
   return fileCount;
 }
@@ -271,7 +280,10 @@ export async function compressZip({
   archiverOptions,
   createRootDirectory,
   rootDirectoryName,
-}: CompressZipOptions) {
+}: CompressZipOptions): Promise<number> {
+  assertAbsolute(source);
+  assertAbsolute(destination);
+
   const output = fs.createWriteStream(destination);
   const archive = archiver('zip', archiverOptions);
   const folder = rootDirectoryName ? rootDirectoryName : source.split(sep).slice(-1)[0];

src/dev/build/lib/integration_tests/fs.test.ts

@@ -12,7 +12,17 @@ import { chmodSync, statSync } from 'fs';
 
 import del from 'del';
 
-import { mkdirp, write, read, getChildPaths, copyAll, getFileHash, untar, gunzip } from '../fs';
+import {
+  mkdirp,
+  write,
+  read,
+  getChildPaths,
+  copyAll,
+  getFileHash,
+  untar,
+  gunzip,
+  compressTar,
+} from '../fs';
 import { getFips } from 'crypto';
 
 const TMP = resolve(__dirname, '../__tmp__');
@@ -356,3 +366,66 @@ describe('gunzip()', () => {
     expect(await read(resolve(destination))).toBe('foo\n');
   });
 });
+
+describe('compressTar()', () => {
+  it('creates a valid .tar.gz that can be extracted with untar', async () => {
+    const tarPath = resolve(TMP, 'test.tar.gz');
+    const fileCount = await compressTar({
+      source: resolve(FIXTURES, 'foo_dir'),
+      destination: tarPath,
+      createRootDirectory: false,
+    });
+
+    expect(fileCount).toBeGreaterThan(0);
+    expect(statSync(tarPath).size).toBeGreaterThan(0);
+
+    const extracted = resolve(TMP, 'extracted');
+    await untar(tarPath, extracted);
+
+    expect(await read(resolve(extracted, 'bar.txt'))).toBe('bar\n');
+    expect(await read(resolve(extracted, 'foo/foo.txt'))).toBe('foo\n');
+  });
+
+  it('wraps contents in a root directory when createRootDirectory is true', async () => {
+    const tarPath = resolve(TMP, 'rooted.tar.gz');
+    await compressTar({
+      source: resolve(FIXTURES, 'foo_dir'),
+      destination: tarPath,
+      createRootDirectory: true,
+    });
+
+    const extracted = resolve(TMP, 'extracted_rooted');
+    await untar(tarPath, extracted);
+
+    expect(await read(resolve(extracted, 'foo_dir/bar.txt'))).toBe('bar\n');
+    expect(await read(resolve(extracted, 'foo_dir/foo/foo.txt'))).toBe('foo\n');
+  });
+
+  it('uses rootDirectoryName when provided', async () => {
+    const tarPath = resolve(TMP, 'custom_root.tar.gz');
+    await compressTar({
+      source: resolve(FIXTURES, 'foo_dir'),
+      destination: tarPath,
+      createRootDirectory: true,
+      rootDirectoryName: 'my-custom-root',
+    });
+
+    const extracted = resolve(TMP, 'extracted_custom');
+    await untar(tarPath, extracted);
+
+    expect(await read(resolve(extracted, 'my-custom-root/bar.txt'))).toBe('bar\n');
+    expect(await read(resolve(extracted, 'my-custom-root/foo/foo.txt'))).toBe('foo\n');
+  });
+
+  it('returns accurate file count', async () => {
+    const tarPath = resolve(TMP, 'count.tar.gz');
+    const fileCount = await compressTar({
+      source: resolve(FIXTURES, 'foo_dir'),
+      destination: tarPath,
+      createRootDirectory: false,
+    });
+
+    // foo_dir contains: .bar, bar.txt, foo/foo.txt
+    expect(fileCount).toBe(3);
+  });
+});

src/dev/build/lib/scan_copy.ts

@@ -14,6 +14,9 @@ import * as Rx from 'rxjs';
 import { assertAbsolute, mkdirp, fsReadDir$ } from './fs';
 import { type DirRecord, type FileRecord, type Record, SomePath } from './fs_records';
 
+// Exclusive create + copy-on-write (reflink) when the filesystem supports it
+const COPY_FLAGS = Fs.constants.COPYFILE_EXCL + Fs.constants.COPYFILE_FICLONE;
+
 interface Options {
   /**
    * directory to copy from
@@ -73,7 +76,7 @@ export async function scanCopy(options: Options) {
         }
 
         return Rx.of(rec);
-      })
+      }, 100)
     );
 
   const handleGenericRec = async (rec: Record) => {
@@ -95,7 +98,7 @@ export async function scanCopy(options: Options) {
       await handleGenericRec(rec);
     }).pipe(
       Rx.mergeMap(() => readDir$(rec)),
-      Rx.mergeMap((ent) => (ent.type === 'dir' ? handleDir$(ent) : handleFile$(ent)))
+      Rx.mergeMap((ent) => (ent.type === 'dir' ? handleDir$(ent) : handleFile$(ent)), 100)
     );
 
   const handleFile$ = (srcRec: FileRecord): Rx.Observable<unknown> =>
@@ -107,7 +110,7 @@ export async function scanCopy(options: Options) {
           flag: 'wx',
         });
       } else {
-        await Fsp.copyFile(rec.source.abs, rec.dest.abs, Fs.constants.COPYFILE_EXCL);
+        await Fsp.copyFile(rec.source.abs, rec.dest.abs, COPY_FLAGS);
       }
 
       await handleGenericRec(rec);

src/dev/build/lib/scan_delete.ts

@@ -68,7 +68,7 @@ export async function scanDelete(directory: string, options: Options) {
     getPathsToDelete$(directory).pipe(
       Rx.mergeMap(
         async (path) => await Fsp.rm(path, { recursive: true, maxRetries: 1 }),
-        options.concurrency
+        options.concurrency ?? 20
       ),
       Rx.count()
     )

src/dev/build/lib/tar_fs.d.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+declare module 'tar-fs' {
+  interface PackOptions {
+    map?: (header: { name: string; type: string }) => { name: string; type: string };
+    filter?: (name: string) => boolean;
+  }
+
+  function pack(cwd: string, opts?: PackOptions): NodeJS.ReadableStream;
+}

src/dev/build/tasks/build_packages_task.ts

@@ -9,10 +9,11 @@
 
 import Path from 'path';
 import * as Fsp from 'fs/promises';
+import { cpus } from 'os';
 
 import * as Peggy from '@kbn/peggy';
 import * as DotText from '@kbn/dot-text';
-import { asyncForEach } from '@kbn/std';
+import { asyncForEachWithLimit } from '@kbn/std';
 import type { TransformConfig } from '@kbn/babel-transform';
 import { withFastAsyncTransform } from '@kbn/babel-transform';
 import { makeMatcher } from '@kbn/picomatcher';
@@ -121,7 +122,7 @@ export const BuildPackages: Task = {
     };
 
     await withFastAsyncTransform(transformConfig, async (transform) => {
-      await asyncForEach(packages, async (pkg) => {
+      await asyncForEachWithLimit(packages, cpus().length, async (pkg) => {
         const allPaths = new Set(Array.from(pkgFileMap.getFiles(pkg), (p) => p.abs));
         const pkgDistPath = build.resolvePath(pkg.normalizedRepoRelativeDir);
         const peggyConfigOutputPaths = new Set<string>();

src/dev/build/tasks/create_archives_task.ts

@@ -49,7 +49,7 @@ export const CreateArchives: Task = {
                 destination,
                 archiverOptions: {
                   zlib: {
-                    level: 9,
+                    level: config.isRelease ? 9 : 6,
                   },
                 },
                 createRootDirectory: true,
@@ -65,12 +65,7 @@ export const CreateArchives: Task = {
               fileCount: await compressTar({
                 source,
                 destination,
-                archiverOptions: {
-                  gzip: true,
-                  gzipOptions: {
-                    level: 9,
-                  },
-                },
+                gzipLevel: config.isRelease ? 9 : 6,
                 createRootDirectory: true,
                 rootDirectoryName: build.getRootDirectory(platform),
               }),