[8.x] @kbn/profiler-cli: collect and display CPU profiles (#216356)

.github/CODEOWNERS

@@ -724,6 +724,7 @@ src/platform/plugins/shared/presentation_util @elastic/kibana-presentation
 x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra
 x-pack/platform/plugins/shared/ai_infra/product_doc_base @elastic/appex-ai-infra
 x-pack/platform/packages/shared/ai-infra/product-doc-common @elastic/appex-ai-infra
+x-pack/platform/packages/shared/kbn-profiler-cli @elastic/obs-knowledge-team
 x-pack/solutions/observability/plugins/profiling_data_access @elastic/obs-ux-infra_services-team
 x-pack/solutions/observability/plugins/profiling @elastic/obs-ux-infra_services-team
 src/platform/packages/shared/kbn-profiling-utils @elastic/obs-ux-infra_services-team

package.json

@@ -741,6 +741,7 @@
     "@kbn/presentation-util-plugin": "link:src/platform/plugins/shared/presentation_util",
     "@kbn/product-doc-base-plugin": "link:x-pack/platform/plugins/shared/ai_infra/product_doc_base",
     "@kbn/product-doc-common": "link:x-pack/platform/packages/shared/ai-infra/product-doc-common",
+    "@kbn/profiler-cli": "link:x-pack/platform/packages/shared/kbn-profiler-cli",
     "@kbn/profiling-data-access-plugin": "link:x-pack/solutions/observability/plugins/profiling_data_access",
     "@kbn/profiling-plugin": "link:x-pack/solutions/observability/plugins/profiling",
     "@kbn/profiling-utils": "link:src/platform/packages/shared/kbn-profiling-utils",
@@ -1109,6 +1110,7 @@
     "chalk": "^4.1.0",
     "cheerio": "^1.0.0-rc.12",
     "chroma-js": "^2.1.0",
+    "chrome-remote-interface": "^0.33.3",
     "classnames": "2.2.6",
     "color": "^4.2.3",
     "commander": "^4.1.1",
@@ -1592,6 +1594,7 @@
     "@types/byte-size": "^8.1.2",
     "@types/chance": "^1.0.0",
     "@types/chroma-js": "^2.1.0",
+    "@types/chrome-remote-interface": "^0.31.14",
     "@types/chromedriver": "^81.0.5",
     "@types/classnames": "^2.2.9",
     "@types/cli-progress": "^3.11.5",
@@ -1778,7 +1781,7 @@
     "eslint-plugin-react-perf": "^3.3.1",
     "eslint-plugin-testing-library": "^7.1.1",
     "eslint-traverse": "^1.0.0",
-    "exit-hook": "^2.2.0",
+    "exit-hook": "2.2.0",
     "expect": "^29.7.0",
     "expose-loader": "^5.0.0",
     "express": "^4.21.2",

renovate.json

@@ -1207,6 +1207,25 @@
       ],
       "minimumReleaseAge": "7 days",
       "enabled": true
+    },
+    {
+      "groupName": "chrome-remote-interface",
+      "matchDepNames": [
+        "chrome-remote-interface",
+        "@types/chrome-remote-interface"
+      ],
+      "reviewers": [
+        "team:obs-knowledge-team"
+      ],
+      "matchBaseBranches": [
+        "main"
+      ],
+      "labels": [
+        "release_note:skip",
+        "backport:all-open"
+      ],
+      "minimumReleaseAge": "7 days",
+      "enabled": true
     }
   ],
   "customManagers": [

scripts/profile.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+require('@babel/register')({
+  extensions: ['.ts', '.js'],
+  presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
+});
+
+require('@kbn/profiler-cli/cli');

src/core/packages/http/server-internal/src/http_server.ts

@@ -101,7 +101,7 @@ function startEluMeasurement<T>(
           active
         )}ms out of ${Math.round(duration)}ms) and ${eluThreshold * 100}% (${Math.round(
           utilization * 100
-        )}%) `,
+        )}%). Run \`node scripts/profile.js\` to find out why.`,
         {
           labels: {
             request_path: path,

src/platform/packages/shared/kbn-tooling-log/src/tooling_log_text_writer.ts

@@ -56,7 +56,7 @@ function shouldWriteType(level: ParsedLogLevel, type: MessageTypes) {
   return Boolean(level.flags[type === 'success' ? 'info' : type]);
 }
 
-function stringifyError(error: string | Error) {
+function stringifyError(error: string | Error): string {
   if (typeof error !== 'string' && !(error instanceof Error)) {
     error = new Error(`"${error}" thrown`);
   }
@@ -65,7 +65,11 @@ function stringifyError(error: string | Error) {
     return error;
   }
 
-  return error.stack || error.message || error;
+  if (error instanceof AggregateError) {
+    return [error.stack, ...error.errors.map(stringifyError)].join('\n');
+  }
+
+  return error.stack || error.message || String(error);
 }
 
 export class ToolingLogTextWriter implements Writer {

tsconfig.base.json

@@ -1442,6 +1442,8 @@
       "@kbn/product-doc-base-plugin/*": ["x-pack/platform/plugins/shared/ai_infra/product_doc_base/*"],
       "@kbn/product-doc-common": ["x-pack/platform/packages/shared/ai-infra/product-doc-common"],
       "@kbn/product-doc-common/*": ["x-pack/platform/packages/shared/ai-infra/product-doc-common/*"],
+      "@kbn/profiler-cli": ["x-pack/platform/packages/shared/kbn-profiler-cli"],
+      "@kbn/profiler-cli/*": ["x-pack/platform/packages/shared/kbn-profiler-cli/*"],
       "@kbn/profiling-data-access-plugin": ["x-pack/solutions/observability/plugins/profiling_data_access"],
       "@kbn/profiling-data-access-plugin/*": ["x-pack/solutions/observability/plugins/profiling_data_access/*"],
       "@kbn/profiling-plugin": ["x-pack/solutions/observability/plugins/profiling"],

x-pack/platform/packages/shared/kbn-profiler-cli/README.md

@@ -0,0 +1,39 @@
+# @kbn/profiler-cli
+
+Profile Kibana while it's running, and open the CPU profile in Speedscope.
+
+## Usage
+
+Run a command by either preceding it with the profiler script:
+`node scripts/profile.js -- $command`
+
+Or by piping it in:
+`$command | node scripts/profile.js`
+
+You can also just run it until SIGINT:
+
+`node scripts/profile.js`
+
+Or with a timeout:
+
+`node scripts/profile.js --timeout=10000`
+
+## Examples
+
+### Commands
+
+You can copy a curl request from the browser, and place it after the command:
+
+`node scripts/profile.js --connections=10 --amount=50 -- curl ...`
+
+You can also use stdin for this, for example:
+
+`pbpaste | node scripts/profile.js`
+
+When using stdin, take into consideration that there is some lag between starting the script and connecting the profiler, so the profiler might miss the first second or so of the running process.
+
+You can also use any other command, like `autocannon`, `sleep` or `xargs`.
+
+### SigInt
+
+By default, the profiler will run until the process exits:`node scripts/profile.js`. This is useful when you have a long running process running separately and you want to collect the profile over a longer time period. Be aware that this might cause memory issues because the profile will get huge. When you press Cmd+C, the profiler will gracefully exit and first write the profile to disk and open Speedscope.

x-pack/platform/packages/shared/kbn-profiler-cli/cli.js

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+require('.').cli();

x-pack/platform/packages/shared/kbn-profiler-cli/index.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { run } from '@kbn/dev-cli-runner';
+import { compact, once, uniq } from 'lodash';
+import { getKibanaProcessId } from './src/get_kibana_process_id';
+import { runCommand } from './src/run_command';
+import { runUntilSigInt } from './src/run_until_sigint';
+import { getProfiler } from './src/get_profiler';
+import { untilStdinCompletes } from './src/until_stdin_completes';
+
+export function cli() {
+  run(
+    async ({ flags, log, addCleanupTask }) => {
+      const pid = flags.pid
+        ? Number(flags.pid)
+        : await getKibanaProcessId({
+            ports: uniq(compact([Number(flags.port), 5603, 5601])),
+          });
+
+      const controller = new AbortController();
+      if (flags.timeout) {
+        setTimeout(() => {
+          controller.abort();
+        }, Number(flags.timeout));
+      }
+
+      process.kill(pid, 'SIGUSR1');
+
+      const stop = once(await getProfiler({ log, type: flags.heap ? 'heap' : 'cpu' }));
+
+      addCleanupTask(() => {
+        // exit-hook, which is used by addCleanupTask,
+        // only allows for synchronous exits, and 3.x
+        // are on ESM which we currently can't use. so
+        // we do a really gross thing where we make
+        // process.exit a noop for a bit until the
+        // profile has been collected and opened
+        const exit = process.exit.bind(process);
+
+        // @ts-expect-error
+        process.exit = () => {};
+
+        stop()
+          .then(() => {
+            exit(0);
+          })
+          .catch((error) => {
+            log.error(error);
+            exit(1);
+          });
+      });
+
+      if (!process.stdin.isTTY) {
+        await untilStdinCompletes();
+      } else if (flags._.length) {
+        const connections = Number(flags.c || flags.connections || 1);
+        const amount = Number(flags.a || flags.amount || 1);
+        const command = flags._;
+
+        log.info(`Executing "${command}" ${amount} times, ${connections} at a time`);
+
+        await runCommand({
+          command,
+          connections,
+          amount,
+          signal: controller.signal,
+        });
+      } else {
+        if (flags.timeout) {
+          log.info(`Awaiting timeout of ${flags.timeout}ms`);
+        } else {
+          log.info(`Awaiting SIGINT (Cmd+C)...`);
+        }
+
+        await runUntilSigInt({
+          log,
+          signal: controller.signal,
+        });
+      }
+
+      await stop();
+    },
+    {
+      flags: {
+        string: ['port', 'pid', 't', 'timeout', 'c', 'connections', 'a', 'amount'],
+        boolean: ['heap'],
+        help: `
+          Usage: node scripts/profiler.js <args> <command>
+
+          --port              Port on which Kibana is running. Falls back to 5603 & 5601.
+          --pid               Process ID to hook into it. Takes precedence over \`port\`.
+          --timeout           Run commands until timeout (in milliseconds)
+          --c, --connections  Number of commands that can be run in parallel.
+          --a, --amount       Amount of times the command should be run
+          --heap              Collect a heap snapshot
+        `,
+        allowUnexpected: false,
+      },
+    }
+  );
+}

x-pack/platform/packages/shared/kbn-profiler-cli/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/x-pack/platform/packages/shared/kbn-profiler-cli'],
+};

x-pack/platform/packages/shared/kbn-profiler-cli/kibana.jsonc

@@ -0,0 +1,7 @@
+{
+  "type": "shared-server",
+  "id": "@kbn/profiler-cli",
+  "owner": "@elastic/obs-knowledge-team",
+  "group": "platform",
+  "visibility": "shared"
+}

x-pack/platform/packages/shared/kbn-profiler-cli/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/profiler-cli",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/platform/packages/shared/kbn-profiler-cli/src/get_kibana_process_id.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import execa from 'execa';
+
+async function getProcessIdAtPort(port: number) {
+  return await execa
+    .command(`lsof -ti :${port}`)
+    .then(({ stdout }) => {
+      return parseInt(stdout.trim().split('\n')[0], 10);
+    })
+    .catch((error) => {
+      return undefined;
+    });
+}
+
+export async function getKibanaProcessId({ ports }: { ports: number[] }): Promise<number> {
+  for (const port of ports) {
+    const pid = await getProcessIdAtPort(port);
+    if (pid) {
+      return pid;
+    }
+  }
+
+  throw new Error(`Kibana process id not found at ports ${ports.join(', ')}`);
+}