Add support for Docker and Serverless to kbn/es

packages/kbn-es/README.mdx

@@ -7,11 +7,13 @@ date: 2022-05-24
 tags: ['kibana', 'dev', 'contributor', 'operations', 'es']
 ---
 
-> A command line utility for running elasticsearch from snapshot, source, archive or even building snapshot artifacts. 
+> A command line utility for running elasticsearch from snapshot, source, archive, docker, serverless or even building snapshot artifacts. 
 
 ## Getting started
 If running elasticsearch from source, elasticsearch needs to be cloned to a sibling directory of Kibana.
 
+If running elasticsearch serverless or a docker container, docker is required to be installed locally. Installation instructions can be found [here](https://www.docker.com/).
+
 To run, go to the Kibana root and run `node scripts/es --help` to get the latest command line options.
 
 The script attempts to preserve the existing interfaces used by Elasticsearch CLI. This includes passing through options with the `-E` argument and the `ES_JAVA_OPTS` environment variable for Java options.
@@ -28,6 +30,16 @@ Run from source with a configured data directory
 node scripts/es source --Epath.data=/home/me/es_data
 ```
 
+Run serverless with a specific image tag
+```
+node scripts/es serverless --tag git-fec36430fba2-x86_64
+```
+
+Run an official Docker release
+```
+node scripts/es docker --tag 8.8.2
+```
+
 ## API
 
 ### run

packages/kbn-es/src/cli_commands/docker.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import dedent from 'dedent';
+import getopts from 'getopts';
+import { ToolingLog } from '@kbn/tooling-log';
+import { getTimeReporter } from '@kbn/ci-stats-reporter';
+
+import { Cluster } from '../cluster';
+import { DOCKER_IMG, DOCKER_REPO, DOCKER_TAG } from '../utils';
+import { Command } from './types';
+
+export const docker: Command = {
+  description: 'Run an Elasticsearch Docker image',
+  usage: 'es docker [<args>]',
+  help: (defaults: Record<string, any> = {}) => {
+    const { password } = defaults;
+
+    return dedent`
+    Options:
+
+      --tag               Image tag of ES to run from ${DOCKER_REPO} [default: ${DOCKER_TAG}]
+      --image             Full path to image of ES to run, has precedence over tag. [default: ${DOCKER_IMG}]
+      --password          Sets password for elastic user [default: ${password}]
+      -E                  Additional key=value settings to pass to Elasticsearch
+      -D                  Override Docker command
+
+    Examples:
+
+      es docker --tag master-SNAPSHOT-amd64
+      es docker --image docker.elastic.co/repo:tag
+      es docker -D 'start es01' 
+    `;
+  },
+  run: async (defaults = {}) => {
+    const runStartTime = Date.now();
+    const log = new ToolingLog({
+      level: 'info',
+      writeTo: process.stdout,
+    });
+    const reportTime = getTimeReporter(log, 'scripts/es docker');
+
+    const argv = process.argv.slice(2);
+    const options = getopts(argv, {
+      alias: {
+        esArgs: 'E',
+        dockerCmd: 'D',
+      },
+
+      string: ['tag', 'image', 'D'],
+
+      default: defaults,
+    });
+
+    const cluster = new Cluster();
+    await cluster.runDocker({
+      reportTime,
+      startTime: runStartTime,
+      ...options,
+    });
+  },
+};

packages/kbn-es/src/cli_commands/index.ts

@@ -10,10 +10,14 @@ import { snapshot } from './snapshot';
 import { source } from './source';
 import { archive } from './archive';
 import { buildSnapshots } from './build_snapshots';
+import { docker } from './docker';
+import { serverless } from './serverless';
 
 export const commands = {
   snapshot,
   source,
   archive,
   build_snapshots: buildSnapshots,
+  docker,
+  serverless,
 };

packages/kbn-es/src/cli_commands/serverless.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import dedent from 'dedent';
+import getopts from 'getopts';
+import { ToolingLog } from '@kbn/tooling-log';
+import { getTimeReporter } from '@kbn/ci-stats-reporter';
+
+import { Cluster } from '../cluster';
+import { SERVERLESS_REPO, SERVERLESS_TAG, SERVERLESS_IMG } from '../utils';
+import { Command } from './types';
+
+export const serverless: Command = {
+  description: 'Run Serverless Elasticsearch through Docker',
+  usage: 'es serverless [<args>]',
+  help: (defaults: Record<string, any> = {}) => {
+    return dedent`
+    Options:
+
+      --tag               Image tag of ES Serverless to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}]
+      --image             Full path of ES Serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}]
+      --clean             Remove existing file system object store before running
+      -E                  Additional key=value settings to pass to Elasticsearch
+
+    Examples:
+
+      es serverless --tag git-fec36430fba2-x86_64
+      es serverless --image docker.elastic.co/repo:tag
+    `;
+  },
+  run: async (defaults = {}) => {
+    const runStartTime = Date.now();
+    const log = new ToolingLog({
+      level: 'info',
+      writeTo: process.stdout,
+    });
+    const reportTime = getTimeReporter(log, 'scripts/es serverless');
+
+    const argv = process.argv.slice(2);
+    const options = getopts(argv, {
+      alias: {
+        basePath: 'base-path',
+        esArgs: 'E',
+      },
+
+      string: ['tag', 'image'],
+      boolean: ['clean'],
+
+      default: defaults,
+    });
+
+    const cluster = new Cluster();
+    await cluster.runServerless({
+      reportTime,
+      startTime: runStartTime,
+      ...options,
+    });
+  },
+};

packages/kbn-es/src/cluster.js

@@ -21,6 +21,8 @@ const {
   extractConfigFiles,
   NativeRealm,
   parseTimeoutToMs,
+  runServerlessCluster,
+  runDockerContainer,
 } = require('./utils');
 const { createCliError } = require('./errors');
 const { promisify } = require('util');
@@ -31,6 +33,8 @@ const { CA_CERT_PATH, ES_NOPASSWORD_P12_PATH, extract } = require('@kbn/dev-util
 const DEFAULT_READY_TIMEOUT = parseTimeoutToMs('1m');
 
 /** @typedef {import('./cluster_exec_options').EsClusterExecOptions} ExecOptions */
+/** @typedef {import('./utils').DockerOptions} DockerOptions */
+/** @typedef {import('./utils').ServerlessOptions}ServerlessrOptions */
 
 // listen to data on stream until map returns anything but undefined
 const first = (stream, map) =>
@@ -467,7 +471,7 @@ exports.Cluster = class Cluster {
       if (stdioTarget) {
         stdioTarget.write(chunk);
       } else {
-        this._log.error(chalk.red());
+        this._log.error(chalk.red(chunk.trim()));
       }
     });
 
@@ -483,7 +487,7 @@ exports.Cluster = class Cluster {
         });
     }
 
-    // observe the exit code of the process and reflect in _outcome promies
+    // observe the exit code of the process and reflect in _outcome promises
     const exitCode = new Promise((resolve) => this._process.once('exit', resolve));
     this._outcome = exitCode.then((code) => {
       if (this._stopCalled) {
@@ -558,4 +562,30 @@ exports.Cluster = class Cluster {
     }
     return esJavaOpts.trim();
   }
+
+  /**
+   * Run an Elasticsearch Serverless Docker cluster
+   *
+   * @param {ServerlessOptions} options
+   */
+  async runServerless(options = {}) {
+    if (this._process || this._outcome) {
+      throw new Error('ES has already been started');
+    }
+
+    await runServerlessCluster(this._log, options);
+  }
+
+  /**
+   * Run an Elasticsearch Docker container
+   *
+   * @param {DockerOptions} options
+   */
+  async runDocker(options = {}) {
+    if (this._process || this._outcome) {
+      throw new Error('ES has already been started');
+    }
+
+    this._process = await runDockerContainer(this._log, options);
+  }
 };

packages/kbn-es/src/cluster_exec_options.ts

@@ -10,7 +10,7 @@ export interface EsClusterExecOptions {
   skipNativeRealmSetup?: boolean;
   reportTime?: (...args: any[]) => void;
   startTime?: number;
-  esArgs?: string[];
+  esArgs?: string[] | string;
   esJavaOpts?: string;
   password?: string;
   skipReadyCheck?: boolean;