Add mock identity provider for serverless

.github/CODEOWNERS

@@ -533,6 +533,7 @@ x-pack/packages/ml/runtime_field_utils @elastic/ml-ui
 x-pack/packages/ml/string_hash @elastic/ml-ui
 x-pack/packages/ml/trained_models_utils @elastic/ml-ui
 x-pack/packages/ml/url_state @elastic/ml-ui
+packages/kbn-mock-idp-plugin @elastic/kibana-security
 packages/kbn-monaco @elastic/appex-sharedux
 x-pack/plugins/monitoring_collection @elastic/obs-ux-infra_services-team
 x-pack/plugins/monitoring @elastic/obs-ux-infra_services-team

package.json

@@ -1221,6 +1221,7 @@
     "@kbn/managed-vscode-config": "link:packages/kbn-managed-vscode-config",
     "@kbn/managed-vscode-config-cli": "link:packages/kbn-managed-vscode-config-cli",
     "@kbn/management-storybook-config": "link:packages/kbn-management/storybook/config",
+    "@kbn/mock-idp-plugin": "link:packages/kbn-mock-idp-plugin",
     "@kbn/openapi-generator": "link:packages/kbn-openapi-generator",
     "@kbn/optimizer": "link:packages/kbn-optimizer",
     "@kbn/optimizer-webpack-helpers": "link:packages/kbn-optimizer-webpack-helpers",

packages/kbn-es/src/cli_commands/serverless.ts

@@ -38,6 +38,7 @@ export const serverless: Command = {
       --host              Publish ES docker container on additional host IP
       --port              The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}]
       --ssl               Enable HTTP SSL on the ES cluster
+      --kibanaUrl         Fully qualified URL where Kibana is hosted (including base path). [default: https://localhost:5601/]
       --skipTeardown      If this process exits, leave the ES cluster running in the background
       --waitForReady      Wait for the ES cluster to be ready to serve requests
       --resources         Overrides resources under ES 'config/' directory, which are by default
@@ -73,7 +74,7 @@ export const serverless: Command = {
         files: 'F',
       },
 
-      string: ['tag', 'image', 'basePath', 'resources', 'host'],
+      string: ['tag', 'image', 'basePath', 'resources', 'host', 'kibanaUrl'],
       boolean: ['clean', 'ssl', 'kill', 'background', 'skipTeardown', 'waitForReady'],
 
       default: defaults,

packages/kbn-es/src/paths.ts

@@ -8,6 +8,7 @@
 
 import Os from 'os';
 import { resolve } from 'path';
+import { REPO_ROOT } from '@kbn/repo-info';
 
 function maybeUseBat(bin: string) {
   return Os.platform().startsWith('win') ? `${bin}.bat` : bin;
@@ -51,6 +52,8 @@ export const SERVERLESS_SECRETS_SSL_PATH = resolve(
 
 export const SERVERLESS_JWKS_PATH = resolve(__dirname, './serverless_resources/jwks.json');
 
+export const SERVERLESS_IDP_METADATA_PATH = resolve(REPO_ROOT, '.es', 'idp_metadata.xml');
+
 export const SERVERLESS_RESOURCES_PATHS = [
   SERVERLESS_OPERATOR_USERS_PATH,
   SERVERLESS_ROLE_MAPPING_PATH,

packages/kbn-es/src/utils/docker.test.ts

@@ -543,6 +543,7 @@ describe('runServerlessEsNode()', () => {
 
 describe('runServerlessCluster()', () => {
   test('should start 3 serverless nodes', async () => {
+    waitUntilClusterReadyMock.mockResolvedValue();
     mockFs({
       [baseEsPath]: {},
     });
@@ -568,6 +569,7 @@ describe('runServerlessCluster()', () => {
   });
 
   test(`should wait for the security index`, async () => {
+    waitUntilClusterReadyMock.mockResolvedValue();
     waitForSecurityIndexMock.mockResolvedValue();
     mockFs({
       [baseEsPath]: {},
@@ -580,6 +582,7 @@ describe('runServerlessCluster()', () => {
   });
 
   test(`should not wait for the security index when security is disabled`, async () => {
+    waitUntilClusterReadyMock.mockResolvedValue();
     mockFs({
       [baseEsPath]: {},
     });

packages/kbn-es/src/utils/docker.ts

@@ -14,12 +14,17 @@ import { Client, ClientOptions, HttpConnection } from '@elastic/elasticsearch';
 
 import { ToolingLog } from '@kbn/tooling-log';
 import { kibanaPackageJson as pkg, REPO_ROOT } from '@kbn/repo-info';
+import { CA_CERT_PATH, ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils';
 import {
-  CA_CERT_PATH,
-  ES_P12_PASSWORD,
-  ES_P12_PATH,
-  kibanaDevServiceAccount,
-} from '@kbn/dev-utils';
+  MOCK_IDP_REALM_NAME,
+  MOCK_IDP_ENTITY_ID,
+  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
+  MOCK_IDP_ATTRIBUTE_ROLES,
+  MOCK_IDP_ATTRIBUTE_EMAIL,
+  MOCK_IDP_ATTRIBUTE_NAME,
+  ensureSAMLRoleMapping,
+  createMockIdpMetadata,
+} from '@kbn/mock-idp-plugin/common';
 
 import { waitForSecurityIndex } from './wait_for_security_index';
 import { createCliError } from '../errors';
@@ -28,6 +33,7 @@ import {
   SERVERLESS_RESOURCES_PATHS,
   SERVERLESS_SECRETS_PATH,
   SERVERLESS_JWKS_PATH,
+  SERVERLESS_IDP_METADATA_PATH,
   SERVERLESS_CONFIG_PATH,
   SERVERLESS_FILES_PATH,
   SERVERLESS_SECRETS_SSL_PATH,
@@ -69,6 +75,8 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions {
   background?: boolean;
   /** Wait for the ES cluster to be ready to serve requests */
   waitForReady?: boolean;
+  /** Fully qualified URL where Kibana is hosted (including base path) */
+  kibanaUrl?: string;
   /**
    * Resource file(s) to overwrite
    * (see list of files that can be overwritten under `packages/kbn-es/src/serverless_resources/users`)
@@ -444,6 +452,50 @@ export function resolveEsArgs(
     DEFAULT_SSL_ESARGS.forEach((arg) => {
       esArgs.set(arg[0], arg[1]);
     });
+
+    // Configure mock identify provider (ES only supports SAML when running in SSL mode)
+    if ('kibanaUrl' in options && options.kibanaUrl) {
+      const trimTrailingSlash = (url: string) => (url.endsWith('/') ? url.slice(0, -1) : url);
+
+      esArgs.set('xpack.security.authc.token.enabled', 'true');
+      esArgs.set(`xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.order`, '0');
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.metadata.path`,
+        `${SERVERLESS_CONFIG_PATH}secrets/idp_metadata.xml`
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.entity_id`,
+        MOCK_IDP_ENTITY_ID
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.entity_id`,
+        trimTrailingSlash(options.kibanaUrl)
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.acs`,
+        `${trimTrailingSlash(options.kibanaUrl)}/api/security/saml/callback`
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.logout`,
+        `${trimTrailingSlash(options.kibanaUrl)}/logout`
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.principal`,
+        MOCK_IDP_ATTRIBUTE_PRINCIPAL
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.groups`,
+        MOCK_IDP_ATTRIBUTE_ROLES
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.name`,
+        MOCK_IDP_ATTRIBUTE_EMAIL
+      );
+      esArgs.set(
+        `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.mail`,
+        MOCK_IDP_ATTRIBUTE_NAME
+      );
+    }
   }
 
   if (customEsArgs) {
@@ -551,6 +603,16 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
     );
   }
 
+  // Create and add meta data for mock identity provider
+  if (options.ssl && options.kibanaUrl) {
+    const metadata = await createMockIdpMetadata(options.kibanaUrl);
+    await Fsp.writeFile(SERVERLESS_IDP_METADATA_PATH, metadata);
+    volumeCmds.push(
+      '--volume',
+      `${SERVERLESS_IDP_METADATA_PATH}:${SERVERLESS_CONFIG_PATH}secrets/idp_metadata.xml:z`
+    );
+  }
+
   volumeCmds.push(
     ...getESp12Volume(),
     ...serverlessResources,
@@ -559,7 +621,6 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
     `${
       ssl ? SERVERLESS_SECRETS_SSL_PATH : SERVERLESS_SECRETS_PATH
     }:${SERVERLESS_CONFIG_PATH}secrets/secrets.json:z`,
-
     '--volume',
     `${SERVERLESS_JWKS_PATH}:${SERVERLESS_CONFIG_PATH}secrets/jwks.json:z`
   );
@@ -661,33 +722,51 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
     process.on('SIGINT', () => teardownServerlessClusterSync(log, options));
   }
 
+  const esNodeUrl = `${options.ssl ? 'https' : 'http'}://${portCmd[1].substring(
+    0,
+    portCmd[1].lastIndexOf(':')
+  )}`;
+
+  const client = getESClient({
+    node: esNodeUrl,
+    auth: {
+      username: ELASTIC_SERVERLESS_SUPERUSER,
+      password: ELASTIC_SERVERLESS_SUPERUSER_PASSWORD,
+    },
+    ...(options.ssl
+      ? {
+          tls: {
+            ca: [fs.readFileSync(CA_CERT_PATH)],
+            // NOTE: Even though we've added ca into the tls options, we are using 127.0.0.1 instead of localhost
+            // for the ip which is not validated. As such we are getting the error
+            // Hostname/IP does not match certificate's altnames: IP: 127.0.0.1 is not in the cert's list:
+            // To work around that we are overriding the function checkServerIdentity too
+            checkServerIdentity: () => {
+              return undefined;
+            },
+          },
+        }
+      : {}),
+  });
+  const readyPromise = waitUntilClusterReady({ client, expectedStatus: 'green', log }).then(() => {
+    if (!options.ssl || !options.kibanaUrl) {
+      return;
+    }
+
+    const roleMappingPromise = ensureSAMLRoleMapping(client);
+
+    log.success(
+      `Created role mapping for mock identity provider. You can now login using ${chalk.bold.cyan(
+        MOCK_IDP_REALM_NAME
+      )} realm`
+    );
+
+    return roleMappingPromise;
+  });
+
   if (options.waitForReady) {
     log.info('Waiting until ES is ready to serve requests...');
-
-    const esNodeUrl = `${options.ssl ? 'https' : 'http'}://${portCmd[1].substring(
-      0,
-      portCmd[1].lastIndexOf(':')
-    )}`;
-
-    const client = getESClient({
-      node: esNodeUrl,
-      auth: { bearer: kibanaDevServiceAccount.token },
-      ...(options.ssl
-        ? {
-            tls: {
-              ca: [fs.readFileSync(CA_CERT_PATH)],
-              // NOTE: Even though we've added ca into the tls options, we are using 127.0.0.1 instead of localhost
-              // for the ip which is not validated. As such we are getting the error
-              // Hostname/IP does not match certificate's altnames: IP: 127.0.0.1 is not in the cert's list:
-              // To work around that we are overriding the function checkServerIdentity too
-              checkServerIdentity: () => {
-                return undefined;
-              },
-            },
-          }
-        : {}),
-    });
-    await waitUntilClusterReady({ client, expectedStatus: 'green', log });
+    await readyPromise;
     if (!options.esArgs || !options.esArgs.includes('xpack.security.enabled=false')) {
       // If security is not disabled, make sure the security index exists before running the test to avoid flakiness
       await waitForSecurityIndex({ client, log });

packages/kbn-es/tsconfig.json

@@ -3,14 +3,22 @@
   "compilerOptions": {
     "outDir": "target/types"
   },
-  "include": ["**/*.ts", "**/*.js", "**/*.json"],
-  "exclude": ["target/**/*"],
+  "include": [
+    "**/*.ts",
+    "**/*.js",
+    "**/*.json"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
   "kbn_references": [
     "@kbn/tooling-log",
     "@kbn/dev-utils",
     "@kbn/dev-proc-runner",
     "@kbn/ci-stats-reporter",
+    "@kbn/mock-idp-plugin",
     "@kbn/jest-serializers",
-    "@kbn/repo-info"
+    "@kbn/repo-info",
+    "@kbn/mock-idp-plugin"
   ]
 }

packages/kbn-mock-idp-plugin/common/constants.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { resolve } from 'path';
+
+export const MOCK_IDP_PLUGIN_PATH = resolve(__dirname, '..');
+export const MOCK_IDP_METADATA_PATH = resolve(MOCK_IDP_PLUGIN_PATH, 'metadata.xml');
+
+export const MOCK_IDP_LOGIN_PATH = '/mock_idp/login';
+export const MOCK_IDP_LOGOUT_PATH = '/mock_idp/logout';
+
+export const MOCK_IDP_REALM_NAME = 'mock-idp';
+export const MOCK_IDP_ENTITY_ID = 'urn:mock-idp'; // Must match `entityID` in `metadata.xml`
+export const MOCK_IDP_ROLE_MAPPING_NAME = 'mock-idp-mapping';
+
+export const MOCK_IDP_ATTRIBUTE_PRINCIPAL = 'http://saml.elastic-cloud.com/attributes/principal';
+export const MOCK_IDP_ATTRIBUTE_ROLES = 'http://saml.elastic-cloud.com/attributes/roles';
+export const MOCK_IDP_ATTRIBUTE_EMAIL = 'http://saml.elastic-cloud.com/attributes/email';
+export const MOCK_IDP_ATTRIBUTE_NAME = 'http://saml.elastic-cloud.com/attributes/name';

packages/kbn-mock-idp-plugin/common/index.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export {
+  MOCK_IDP_PLUGIN_PATH,
+  MOCK_IDP_METADATA_PATH,
+  MOCK_IDP_LOGIN_PATH,
+  MOCK_IDP_LOGOUT_PATH,
+  MOCK_IDP_REALM_NAME,
+  MOCK_IDP_ENTITY_ID,
+  MOCK_IDP_ROLE_MAPPING_NAME,
+  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
+  MOCK_IDP_ATTRIBUTE_ROLES,
+  MOCK_IDP_ATTRIBUTE_EMAIL,
+  MOCK_IDP_ATTRIBUTE_NAME,
+} from './constants';
+export {
+  createMockIdpMetadata,
+  createSAMLResponse,
+  ensureSAMLRoleMapping,
+  parseSAMLAuthnRequest,
+} from './utils';