feat(security,serverless): enable UIAM mode by default

src/cli/serve/serve.js

@@ -279,8 +279,8 @@ export default function (program) {
         'Collect more complete stack traces. See src/cli/dev.js for explanation.'
       )
       .option(
-        '--uiam',
-        'Configure Kibana with Universal Identity and Access Management (UIAM) support when running in serverless project mode.'
+        '--no-uiam',
+        'Prevents configuring Kibana with Universal Identity and Access Management (UIAM) support when running in serverless project mode.'
       );
   }
 
@@ -323,7 +323,7 @@ export default function (program) {
       cache: !!opts.cache,
       dist: !!opts.dist,
       serverless: isServerlessMode,
-      uiam: isServerlessSamlSupported && !!opts.uiam,
+      uiam: isServerlessSamlSupported && opts.uiam !== false,
     };
 
     // In development mode, the main process uses the @kbn/dev-cli-mode
@@ -442,7 +442,7 @@ function tryConfigureServerlessSamlProvider(rawConfig, opts, extraCliOptions) {
     });
   }
 
-  if (opts.uiam && DEV_UTILS_SUPPORTED) {
+  if (opts.uiam !== false && DEV_UTILS_SUPPORTED) {
     // Ensure the key/cert pair is loaded dynamically to exclude it from the production build.
     // eslint-disable-next-line import/no-dynamic-require
     const { KBN_CERT_PATH, KBN_KEY_PATH } = require(DEV_UTILS_PATH);

src/cli/serve/serve.test.js

@@ -83,15 +83,22 @@ describe('applyConfigOverrides', () => {
     });
   });
 
-  it('alters config to enable SAML Mock IdP in serverless dev mode', () => {
-    expect(applyConfigOverrides({}, { dev: true, serverless: true }, {}, {})).toEqual({
+  it('alters config to enable SAML Mock IdP and UIAM in serverless dev mode', () => {
+    expect(applyConfigOverrides({}, { dev: true, serverless: true, uiam: true }, {}, {})).toEqual({
       elasticsearch: {
         hosts: ['https://localhost:9200'],
         serviceAccountToken: kibanaDevServiceAccount.token,
         ssl: { certificateAuthorities: expect.stringContaining('ca.crt') },
       },
+      mockIdpPlugin: { uiam: { enabled: true } },
       plugins: { paths: [] },
       xpack: {
+        cloud: {
+          id: 'local-dev:ZG9ja2VyLmludGVybmFsOjkyMDAkaG9zdDo5MjAwJGtpYmFuYTo5MjAw',
+          organization_id: 'org1234567890',
+          projects_url: '',
+          serverless: { project_id: 'abcdef12345678901234567890123456' },
+        },
         security: {
           authc: {
             providers: {
@@ -108,27 +115,30 @@ describe('applyConfigOverrides', () => {
             },
             selector: { enabled: false },
           },
+          uiam: {
+            enabled: true,
+            sharedSecret: 'Dw7eRt5yU2iO9pL3aS4dF6gH8jK0lZ1xC2vB3nM4qW5=',
+            url: 'https://localhost:8443',
+            ssl: {
+              certificate: KBN_CERT_PATH,
+              key: KBN_KEY_PATH,
+              verificationMode: 'none',
+            },
+          },
         },
       },
     });
   });
 
-  it('alters config to enable UIAM if `--uiam` flag is passed in serverless dev mode', () => {
-    expect(applyConfigOverrides({}, { dev: true, serverless: true, uiam: true }, {}, {})).toEqual({
+  it('omits UIAM config if `--no-uiam` flag is passed in serverless dev mode', () => {
+    expect(applyConfigOverrides({}, { dev: true, serverless: true, uiam: false }, {}, {})).toEqual({
       elasticsearch: {
         hosts: ['https://localhost:9200'],
         serviceAccountToken: kibanaDevServiceAccount.token,
         ssl: { certificateAuthorities: expect.stringContaining('ca.crt') },
       },
-      mockIdpPlugin: { uiam: { enabled: true } },
       plugins: { paths: [] },
       xpack: {
-        cloud: {
-          id: 'local-dev:ZG9ja2VyLmludGVybmFsOjkyMDAkaG9zdDo5MjAwJGtpYmFuYTo5MjAw',
-          organization_id: 'org1234567890',
-          projects_url: '',
-          serverless: { project_id: 'abcdef12345678901234567890123456' },
-        },
         security: {
           authc: {
             providers: {
@@ -145,16 +155,6 @@ describe('applyConfigOverrides', () => {
             },
             selector: { enabled: false },
           },
-          uiam: {
-            enabled: true,
-            sharedSecret: 'Dw7eRt5yU2iO9pL3aS4dF6gH8jK0lZ1xC2vB3nM4qW5=',
-            url: 'https://localhost:8443',
-            ssl: {
-              certificate: KBN_CERT_PATH,
-              key: KBN_KEY_PATH,
-              verificationMode: 'none',
-            },
-          },
         },
       },
     });

src/platform/packages/private/kbn-mock-idp-utils/src/constants.ts

@@ -37,6 +37,13 @@ export const MOCK_IDP_UIAM_ORGANIZATION_ID = 'org1234567890';
 export const MOCK_IDP_UIAM_PROJECT_ID = 'abcdef12345678901234567890123456';
 export const MOCK_IDP_UIAM_PROJECT_ID2 = 'fedcba65432109876543210987654321';
 
+// cloud.id is decoded by the security plugin to obtain the ES endpoint for UIAM API key conversion.
+// CI:    decodes to https://es01:9220 (ES listens on port 9220 inside the Docker network)
+// Local: decodes to https://host.docker.internal:9220 (ES is on the host, reached via Docker bridge)
+export const MOCK_IDP_UIAM_CLOUD_ID = process.env.CI
+  ? 'ci:ZXMwMTo5MjIwJDo5MjIwJGtpYmFuYTo5MjIw'
+  : 'local-dev:ZG9ja2VyLmludGVybmFsOjkyMjAkaG9zdDo5MjIwJGtpYmFuYTo5MjIw';
+
 // Sometimes it is useful or required to point local UIAM service clients, or clients operating within the same Docker
 // network (i.e., Elasticsearch), to a different UIAM service URL. For example, http://host.docker.internal:8080 can be
 // used to route requests through the host network, making it easier to capture traffic with a network analyzer running

src/platform/packages/private/kbn-mock-idp-utils/src/index.ts

@@ -35,6 +35,7 @@ export {
   MOCK_IDP_UIAM_SHARED_SECRET,
   MOCK_IDP_UIAM_ORG_ADMIN_API_KEY,
   MOCK_IDP_UIAM_ORGANIZATION_ID,
+  MOCK_IDP_UIAM_CLOUD_ID,
   MOCK_IDP_UIAM_PROJECT_ID,
   MOCK_IDP_UIAM_PROJECT_ID2,
 } from './constants';

src/platform/packages/shared/kbn-es/src/cli_commands/serverless.ts

@@ -64,7 +64,7 @@ export const serverless: Command = {
                           ${SERVERLESS_RESOURCES_PATHS.map((filePath) => basename(filePath)).join(
                             ' | '
                           )}
-      --uiam              Configure ES serverless with Universal Identity and Access Management (UIAM) support.
+      --uiam              Configure ES serverless with Universal Identity and Access Management (UIAM) support [default: true].
 
       -E                  Additional key=value settings to pass to ES
       -F                  Absolute paths for files to mount into containers
@@ -118,6 +118,7 @@ export const serverless: Command = {
         kibanaUrl: 'http://localhost:5601/',
         dataPath: 'stateless',
         ssl: true,
+        uiam: true,
       },
     }) as unknown as ServerlessOptions;
 

src/platform/packages/shared/kbn-es/src/utils/docker_uiam.test.ts

@@ -190,6 +190,8 @@ describe(`#runUiamContainer()`, () => {
             "--env",
             "quarkus.log.category.\\"co.elastic.cloud.uiam\\".level=DEBUG",
             "--env",
+            "quarkus.log.category.\\"co.elastic.cloud.uiam.app.authentication.ClientCertificateExtractor\\".level=INFO",
+            "--env",
             "quarkus.log.console.json.enabled=false",
             "--env",
             "quarkus.log.level=INFO",

src/platform/packages/shared/kbn-es/src/utils/docker_uiam.ts

@@ -154,6 +154,8 @@ export const UIAM_CONTAINERS = [
       '--env',
       `quarkus.log.category."co.elastic.cloud.uiam".level=${env.UIAM_APP_LOGGING_LEVEL}`,
       '--env',
+      `quarkus.log.category."co.elastic.cloud.uiam.app.authentication.ClientCertificateExtractor".level=${env.UIAM_LOGGING_LEVEL}`,
+      '--env',
       'quarkus.log.console.json.enabled=false',
       '--env',
       `quarkus.log.level=${env.UIAM_LOGGING_LEVEL}`,

src/platform/packages/shared/kbn-ftr-common-functional-services/services/saml_auth/saml_auth_provider.ts

@@ -12,6 +12,8 @@ import { SamlSessionManager } from '@kbn/test';
 import expect from '@kbn/expect';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { resolve } from 'path';
+import getopts from 'getopts';
+import type { ServerlessProjectType } from '@kbn/es';
 import type { FtrProviderContext } from '../ftr_provider_context';
 import { getAuthProvider } from './get_auth_provider';
 import type { InternalRequestHeader } from './default_request_headers';
@@ -53,6 +55,11 @@ export function SamlAuthProvider({ getService }: FtrProviderContext) {
   const customRolesFileName = process.env.ROLES_FILENAME_OVERRIDE;
   const cloudUsersFilePath = resolve(REPO_ROOT, '.ftr', customRolesFileName ?? 'role_users.json');
 
+  const kbnServerOptions = getopts(config.get('kbnTestServer.serverArgs'), {
+    boolean: ['xpack.security.uiam.enabled'],
+    string: ['serverless', 'xpack.cloud.organization_id'],
+  });
+
   // Sharing the instance within FTR config run means cookies are persistent for each role between tests.
   const sessionManager = new SamlSessionManager({
     hostOptions: {
@@ -69,6 +76,13 @@ export function SamlAuthProvider({ getService }: FtrProviderContext) {
       sourcePath: authRoleProvider.getRolesDefinitionPath(),
     },
     cloudUsersFilePath,
+    serverless: !!kbnServerOptions.serverless
+      ? {
+          uiam: kbnServerOptions['xpack.security.uiam.enabled'] ?? false,
+          projectType: kbnServerOptions.serverless as ServerlessProjectType,
+          organizationId: kbnServerOptions['xpack.cloud.organization_id']!,
+        }
+      : undefined,
   });
 
   const DEFAULT_ROLE = authRoleProvider.getDefaultRole();

src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/automatic_import/serverless/security_complete.serverless.config.ts

@@ -12,7 +12,6 @@ import type { ScoutServerConfig } from '../../../../../types';
 
 export const servers: ScoutServerConfig = {
   ...defaultConfig,
-  esServerlessOptions: { uiam: true },
   kbnTestServer: {
     ...defaultConfig.kbnTestServer,
     serverArgs: [

src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/default/serverless/serverless.base.config.ts

@@ -11,16 +11,19 @@ import { resolve, join } from 'path';
 import { format as formatUrl } from 'url';
 import Fs from 'fs';
 
-import { CA_CERT_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils';
+import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils';
 import {
   fleetPackageRegistryDockerImage,
   defineDockerServersConfig,
 } from '@kbn/test-docker-servers';
 import { getDockerFileMountPath } from '@kbn/es';
 import {
   MOCK_IDP_REALM_NAME,
+  MOCK_IDP_UIAM_CLOUD_ID,
   MOCK_IDP_UIAM_ORGANIZATION_ID,
   MOCK_IDP_UIAM_PROJECT_ID,
+  MOCK_IDP_UIAM_SERVICE_URL,
+  MOCK_IDP_UIAM_SHARED_SECRET,
 } from '@kbn/mock-idp-utils';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { ScoutServerConfig } from '../../../../../types';
@@ -37,6 +40,9 @@ const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/
  */
 const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
 
+// Indicates whether the config is used on CI or locally.
+const isRunOnCI = process.env.CI;
+
 const servers = {
   elasticsearch: {
     protocol: 'https',
@@ -94,6 +100,7 @@ export const defaultConfig: ScoutServerConfig = {
     ],
     ssl: true, // SSL is required for SAML realm
   },
+  esServerlessOptions: { uiam: true },
   kbnTestServer: {
     buildArgs: [],
     env: {
@@ -150,7 +157,7 @@ export const defaultConfig: ScoutServerConfig = {
       '--xpack.cloud.base_url=https://fake-cloud.elastic.co',
       '--xpack.cloud.billing_url=/billing/overview/',
       '--xpack.cloud.deployments_url=/deployments',
-      '--xpack.cloud.id=ftr_fake_cloud_id',
+      `--xpack.cloud.id=${MOCK_IDP_UIAM_CLOUD_ID}`,
       `--xpack.cloud.organization_id=${MOCK_IDP_UIAM_ORGANIZATION_ID}`,
       '--xpack.cloud.organization_url=/account/',
       '--xpack.cloud.profile_url=/user/settings/',
@@ -194,6 +201,13 @@ export const defaultConfig: ScoutServerConfig = {
           ],
         },
       ])}`,
+      ...(isRunOnCI ? [] : ['--mockIdpPlugin.uiam.enabled=true']),
+      `--xpack.security.uiam.enabled=true`,
+      `--xpack.security.uiam.url=${MOCK_IDP_UIAM_SERVICE_URL}`,
+      `--xpack.security.uiam.sharedSecret=${MOCK_IDP_UIAM_SHARED_SECRET}`,
+      `--xpack.security.uiam.ssl.certificate=${KBN_CERT_PATH}`,
+      `--xpack.security.uiam.ssl.key=${KBN_KEY_PATH}`,
+      '--xpack.security.uiam.ssl.verificationMode=none',
     ],
   },
 };

src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/uiam_local/serverless/observability_complete.serverless.config.ts

@@ -7,36 +7,23 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { MOCK_IDP_UIAM_SERVICE_URL, MOCK_IDP_UIAM_SHARED_SECRET } from '@kbn/mock-idp-utils';
-import { KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { resolve } from 'path';
+import { REPO_ROOT } from '@kbn/repo-info';
 import { servers as defaultConfig } from '../../default/serverless/observability_complete.serverless.config';
 import type { ScoutServerConfig } from '../../../../../types';
 
-// Indicates whether the config is used on CI or locally.
-const isRunOnCI = process.env.CI;
+// We need to test certain APIs that are only exposed by the plugin contract and not through
+// any HTTP endpoint, so this test plugin exposes these APIs through test HTTP endpoints that
+// we can call in our tests.
+const pluginPath = `--plugin-path=${resolve(
+  REPO_ROOT,
+  'x-pack/platform/test/security_functional/plugins/test_endpoints'
+)}`;
 
 export const servers: ScoutServerConfig = {
   ...defaultConfig,
-  esServerlessOptions: { uiam: true },
   kbnTestServer: {
     ...defaultConfig.kbnTestServer,
-    serverArgs: [
-      ...defaultConfig.kbnTestServer.serverArgs,
-      ...(isRunOnCI ? [] : ['--mockIdpPlugin.uiam.enabled=true']),
-      `--xpack.security.uiam.enabled=true`,
-      `--xpack.security.uiam.url=${MOCK_IDP_UIAM_SERVICE_URL}`,
-      `--xpack.security.uiam.sharedSecret=${MOCK_IDP_UIAM_SHARED_SECRET}`,
-      `--xpack.security.uiam.ssl.certificate=${KBN_CERT_PATH}`,
-      `--xpack.security.uiam.ssl.key=${KBN_KEY_PATH}`,
-      '--xpack.security.uiam.ssl.verificationMode=none',
-      // cloud.id is decoded by the security plugin to obtain the ES endpoint for UIAM API key conversion.
-      // CI:    decodes to https://es01:9220 (ES listens on port 9220 inside the Docker network)
-      // Local: decodes to https://host.docker.internal:9220 (ES is on the host, reached via Docker bridge)
-      `--xpack.cloud.id=${
-        isRunOnCI
-          ? 'ci:ZXMwMTo5MjIwJDo5MjIwJGtpYmFuYTo5MjIw'
-          : 'local-dev:ZG9ja2VyLmludGVybmFsOjkyMjAkaG9zdDo5MjIwJGtpYmFuYTo5MjIw'
-      }`,
-    ],
+    serverArgs: [...defaultConfig.kbnTestServer.serverArgs, pluginPath],
   },
 };

src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/uiam_local/serverless/observability_logs_essentials.serverless.config.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { resolve } from 'path';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { servers as defaultConfig } from '../../default/serverless/observability_logs_essentials.serverless.config';
+import type { ScoutServerConfig } from '../../../../../types';
+
+// We need to test certain APIs that are only exposed by the plugin contract and not through
+// any HTTP endpoint, so this test plugin exposes these APIs through test HTTP endpoints that
+// we can call in our tests.
+const pluginPath = `--plugin-path=${resolve(
+  REPO_ROOT,
+  'x-pack/platform/test/security_functional/plugins/test_endpoints'
+)}`;
+
+export const servers: ScoutServerConfig = {
+  ...defaultConfig,
+  kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
+    serverArgs: [...defaultConfig.kbnTestServer.serverArgs, pluginPath],
+  },
+};

src/platform/packages/shared/kbn-scout/src/servers/configs/config_sets/uiam_local/serverless/search.serverless.config.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { resolve } from 'path';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { servers as defaultConfig } from '../../default/serverless/search.serverless.config';
+import type { ScoutServerConfig } from '../../../../../types';
+
+// We need to test certain APIs that are only exposed by the plugin contract and not through
+// any HTTP endpoint, so this test plugin exposes these APIs through test HTTP endpoints that
+// we can call in our tests.
+const pluginPath = `--plugin-path=${resolve(
+  REPO_ROOT,
+  'x-pack/platform/test/security_functional/plugins/test_endpoints'
+)}`;
+
+export const servers: ScoutServerConfig = {
+  ...defaultConfig,
+  kbnTestServer: {
+    ...defaultConfig.kbnTestServer,
+    serverArgs: [...defaultConfig.kbnTestServer.serverArgs, pluginPath],
+  },
+};