[FTR] add feature flags configs for deployment-agnostic tests

.buildkite/ftr_base_serverless_configs.yml

@@ -3,6 +3,7 @@ disabled:
 
   # Serverless deployment-agnostic default config for api-integration tests
   - x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts
   # Serverless base config files
   - x-pack/test_serverless/api_integration/config.base.ts
   - x-pack/test_serverless/functional/config.base.ts

.buildkite/ftr_oblt_serverless_configs.yml

@@ -39,3 +39,4 @@ enabled:
   - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.apm.serverless.config.ts
   - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.synthetics.serverless.config.ts
   - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.streams.serverless.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/feature_flag_configs/serverless/oblt.synthetics.serverless.config.ts

.buildkite/ftr_oblt_stateful_configs.yml

@@ -48,3 +48,4 @@ enabled:
   - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
   - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.apm.stateful.config.ts
   - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/feature_flag_configs/stateful/oblt.synthetics.stateful.config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -1,6 +1,8 @@
 disabled:
   # Stateful base config for deployment-agnostic tests
   - x-pack/test/api_integration/deployment_agnostic/default_configs/stateful.config.base.ts
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts
+
   # Base config files, only necessary to inform config finding script
   - src/platform/test/functional/config.base.js
   - src/platform/test/functional/firefox/config.base.ts

x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/index.ts

@@ -12,10 +12,5 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
     loadTestFile(require.resolve('./burn_rate_rule'));
     loadTestFile(require.resolve('./es_query'));
     loadTestFile(require.resolve('./custom_threshold'));
-
-    describe('Synthetics Alerting', () => {
-      loadTestFile(require.resolve('./synthetics/synthetics_default_rule'));
-      loadTestFile(require.resolve('./synthetics/custom_status_rule'));
-    });
   });
 }

x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  fleetPackageRegistryDockerImage,
+  FtrConfigProviderContext,
+  Config,
+  defineDockerServersConfig,
+} from '@kbn/test';
+
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
+import { ServerlessProjectType } from '@kbn/es';
+import path from 'path';
+import { DeploymentAgnosticCommonServices, services } from '../services';
+import { LOCAL_PRODUCT_DOC_PATH } from './common_paths';
+import { updateKbnServerArguments } from './helpers';
+
+interface CreateTestConfigOptions<T extends DeploymentAgnosticCommonServices> {
+  serverlessProject: ServerlessProjectType;
+  esServerArgs?: string[];
+  kbnServerArgs?: string[];
+  services?: T;
+  testFiles: string[];
+  junit: { reportName: string };
+  suiteTags?: { include?: string[]; exclude?: string[] };
+}
+
+// include settings from elasticsearch controller
+// https://github.com/elastic/elasticsearch-controller/blob/main/helm/values.yaml
+const esServerArgsFromController = {
+  es: [],
+  oblt: [
+    'xpack.apm_data.enabled=true',
+    // for ML, data frame analytics are not part of this project type
+    'xpack.ml.dfa.enabled=false',
+  ],
+  security: ['xpack.security.authc.api_key.cache.max_keys=70000'],
+  chat: [],
+};
+
+// include settings from kibana controller
+// https://github.com/elastic/kibana-controller/blob/main/internal/controllers/kibana/config/config_settings.go
+const kbnServerArgsFromController = {
+  es: [
+    // useful for testing (also enabled in MKI QA)
+    '--coreApp.allowDynamicConfigOverrides=true',
+  ],
+  oblt: [
+    '--coreApp.allowDynamicConfigOverrides=true',
+    // defined in MKI control plane
+    '--xpack.uptime.service.manifestUrl=mockDevUrl',
+  ],
+  security: [
+    '--coreApp.allowDynamicConfigOverrides=true',
+    // disable fleet task that writes to metrics.fleet_server.* data streams, impacting functional tests
+    `--xpack.task_manager.unsafe.exclude_task_types=${JSON.stringify(['Fleet-Metrics-Task'])}`,
+  ],
+  chat: [],
+};
+
+export function createServerlessFeatureFlagTestConfig<T extends DeploymentAgnosticCommonServices>(
+  options: CreateTestConfigOptions<T>
+) {
+  return async ({ readConfigFile }: FtrConfigProviderContext): Promise<Config> => {
+    const packageRegistryConfig = path.join(__dirname, './fixtures/package_registry_config.yml');
+    const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
+    let kbnServerArgs: string[] = [];
+
+    if (options.kbnServerArgs) {
+      kbnServerArgs = await updateKbnServerArguments(options.kbnServerArgs);
+    }
+
+    /**
+     * This is used by CI to set the docker registry port
+     * you can also define this environment variable locally when running tests which
+     * will spin up a local docker package registry locally for you
+     * if this is defined it takes precedence over the `packageRegistryOverride` variable
+     */
+    const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
+
+    const svlSharedConfig = await readConfigFile(
+      require.resolve('@kbn/test-suites-serverless/shared/config.base')
+    );
+
+    return {
+      ...svlSharedConfig.getAll(),
+
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
+      services: {
+        // services can be customized, but must extend DeploymentAgnosticCommonServices
+        ...(options.services || services),
+      },
+      dockerServers: defineDockerServersConfig({
+        registry: {
+          enabled: !!dockerRegistryPort,
+          image: fleetPackageRegistryDockerImage,
+          portInContainer: 8080,
+          port: dockerRegistryPort,
+          args: dockerArgs,
+          waitForLogLine: 'package manifests loaded',
+          waitForLogLineTimeoutMs: 60 * 2 * 1000, // 2 minutes
+        },
+      }),
+      esTestCluster: {
+        ...svlSharedConfig.get('esTestCluster'),
+        serverArgs: [
+          ...svlSharedConfig.get('esTestCluster.serverArgs'),
+          // custom native roles are enabled only for search and security projects
+          ...(options.serverlessProject !== 'oblt'
+            ? ['xpack.security.authc.native_roles.enabled=true']
+            : []),
+          ...esServerArgsFromController[options.serverlessProject],
+          ...(options.esServerArgs || []),
+        ],
+      },
+      kbnTestServer: {
+        ...svlSharedConfig.get('kbnTestServer'),
+        serverArgs: [
+          ...svlSharedConfig.get('kbnTestServer.serverArgs'),
+          ...kbnServerArgsFromController[options.serverlessProject],
+          `--serverless=${options.serverlessProject}`,
+          ...(options.serverlessProject === 'oblt'
+            ? [
+                // defined in MKI control plane. Necessary for Synthetics app testing
+                '--xpack.uptime.service.password=test',
+                '--xpack.uptime.service.username=localKibanaIntegrationTestsUser',
+                '--xpack.uptime.service.devUrl=mockDevUrl',
+                '--xpack.uptime.service.manifestUrl=mockDevUrl',
+                `--xpack.productDocBase.artifactRepositoryUrl=file:///${LOCAL_PRODUCT_DOC_PATH}`,
+              ]
+            : []),
+          ...(dockerRegistryPort
+            ? [`--xpack.fleet.registryUrl=http://localhost:${dockerRegistryPort}`]
+            : []),
+          ...kbnServerArgs,
+        ],
+      },
+      testFiles: options.testFiles,
+      junit: options.junit,
+      suiteTags: {
+        include: options.suiteTags?.include,
+        exclude: [...(options.suiteTags?.exclude || []), 'skipServerless'],
+      },
+    };
+  };
+}

x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts

@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  MOCK_IDP_REALM_NAME,
+  MOCK_IDP_ENTITY_ID,
+  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
+  MOCK_IDP_ATTRIBUTE_ROLES,
+  MOCK_IDP_ATTRIBUTE_EMAIL,
+  MOCK_IDP_ATTRIBUTE_NAME,
+} from '@kbn/mock-idp-utils';
+import {
+  fleetPackageRegistryDockerImage,
+  esTestConfig,
+  kbnTestConfig,
+  systemIndicesSuperuser,
+  FtrConfigProviderContext,
+  defineDockerServersConfig,
+} from '@kbn/test';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
+import path from 'path';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { STATEFUL_ROLES_ROOT_PATH } from '@kbn/es';
+import { DeploymentAgnosticCommonServices, services } from '../services';
+import { AI_ASSISTANT_SNAPSHOT_REPO_PATH, LOCAL_PRODUCT_DOC_PATH } from './common_paths';
+import { updateKbnServerArguments } from './helpers';
+
+interface CreateTestConfigOptions<T extends DeploymentAgnosticCommonServices> {
+  esServerArgs?: string[];
+  kbnServerArgs?: string[];
+  services?: T;
+  testFiles: string[];
+  junit: { reportName: string };
+  suiteTags?: { include?: string[]; exclude?: string[] };
+}
+
+export function createStatefulFeatureFlagTestConfig<T extends DeploymentAgnosticCommonServices>(
+  options: CreateTestConfigOptions<T>
+) {
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    // if config is executed on CI or locally
+    const isRunOnCI = process.env.CI;
+
+    const packageRegistryConfig = path.join(__dirname, './fixtures/package_registry_config.yml');
+    const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
+    let kbnServerArgs: string[] = [];
+
+    if (options.kbnServerArgs) {
+      kbnServerArgs = await updateKbnServerArguments(options.kbnServerArgs);
+    }
+
+    /**
+     * This is used by CI to set the docker registry port
+     * you can also define this environment variable locally when running tests which
+     * will spin up a local docker package registry locally for you
+     * if this is defined it takes precedence over the `packageRegistryOverride` variable
+     */
+    const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
+
+    const xPackAPITestsConfig = await readConfigFile(require.resolve('../../config.ts'));
+
+    // TODO: move to kbn-es because currently metadata file has hardcoded entityID and Location
+    const idpPath = require.resolve(
+      '@kbn/security-api-integration-helpers/saml/idp_metadata_mock_idp.xml'
+    );
+    const samlIdPPlugin = path.resolve(
+      __dirname,
+      '../../../security_api_integration/plugins/saml_provider'
+    );
+
+    const servers = {
+      kibana: {
+        ...kbnTestConfig.getUrlParts(systemIndicesSuperuser),
+        protocol: process.env.TEST_CLOUD ? 'https' : 'http',
+      },
+      elasticsearch: {
+        ...esTestConfig.getUrlParts(),
+        protocol: process.env.TEST_CLOUD ? 'https' : 'http',
+      },
+    };
+
+    const kbnUrl = `${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`;
+
+    return {
+      servers,
+      testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
+      dockerServers: defineDockerServersConfig({
+        registry: {
+          enabled: !!dockerRegistryPort,
+          image: fleetPackageRegistryDockerImage,
+          portInContainer: 8080,
+          port: dockerRegistryPort,
+          args: dockerArgs,
+          waitForLogLine: 'package manifests loaded',
+          waitForLogLineTimeoutMs: 60 * 2 * 1000, // 2 minutes
+        },
+      }),
+      testFiles: options.testFiles,
+      security: { disableTestUser: true },
+      // services can be customized, but must extend DeploymentAgnosticCommonServices
+      services: options.services || services,
+      junit: options.junit,
+      suiteTags: {
+        include: options.suiteTags?.include,
+        exclude: [...(options.suiteTags?.exclude || []), 'skipStateful'],
+      },
+
+      esTestCluster: {
+        ...xPackAPITestsConfig.get('esTestCluster'),
+        serverArgs: [
+          ...xPackAPITestsConfig.get('esTestCluster.serverArgs'),
+          'xpack.security.authc.token.enabled=true',
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.order=0`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.metadata.path=${idpPath}`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.entity_id=${MOCK_IDP_ENTITY_ID}`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.entity_id=${kbnUrl}`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.acs=${kbnUrl}/api/security/saml/callback`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.logout=${kbnUrl}/logout`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.principal=${MOCK_IDP_ATTRIBUTE_PRINCIPAL}`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.groups=${MOCK_IDP_ATTRIBUTE_ROLES}`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.name=${MOCK_IDP_ATTRIBUTE_NAME}`,
+          `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.mail=${MOCK_IDP_ATTRIBUTE_EMAIL}`,
+          `path.repo=${AI_ASSISTANT_SNAPSHOT_REPO_PATH}`,
+          ...(options.esServerArgs || []),
+        ],
+        files: [
+          // Passing the roles that are equivalent to the ones we have in serverless
+          path.resolve(REPO_ROOT, STATEFUL_ROLES_ROOT_PATH, 'roles.yml'),
+        ],
+      },
+      kbnTestServer: {
+        ...xPackAPITestsConfig.get('kbnTestServer'),
+        serverArgs: [
+          ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'),
+          // if the config is run locally, explicitly enable mock-idp-plugin for UI role selector
+          ...(isRunOnCI ? [] : ['--mock_idp_plugin.enabled=true']),
+          // This ensures that we register the Security SAML API endpoints.
+          // In the real world the SAML config is injected by control plane.
+          `--plugin-path=${samlIdPPlugin}`,
+          '--xpack.cloud.id=ftr_fake_cloud_id',
+          // Ensure that SAML is used as the default authentication method whenever a user navigates to Kibana. In other
+          // words, Kibana should attempt to authenticate the user using the provider with the lowest order if the Login
+          // Selector is disabled (replicating Serverless configuration). By declaring `cloud-basic` with a higher
+          // order, we indicate that basic authentication can still be used, but only if explicitly requested when the
+          // user navigates to `/login` page directly and enters username and password in the login form.
+          '--xpack.security.authc.selector.enabled=false',
+          `--xpack.security.authc.providers=${JSON.stringify({
+            saml: { 'cloud-saml-kibana': { order: 0, realm: MOCK_IDP_REALM_NAME } },
+            basic: { 'cloud-basic': { order: 1 } },
+          })}`,
+          `--server.publicBaseUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`,
+          '--xpack.uptime.service.password=test',
+          '--xpack.uptime.service.username=localKibanaIntegrationTestsUser',
+          '--xpack.uptime.service.devUrl=mockDevUrl',
+          '--xpack.uptime.service.manifestUrl=mockDevUrl',
+          '--xpack.observabilityAIAssistant.disableKbSemanticTextMigration=true',
+          `--xpack.productDocBase.artifactRepositoryUrl=file:///${LOCAL_PRODUCT_DOC_PATH}`,
+          ...(dockerRegistryPort
+            ? [`--xpack.fleet.registryUrl=http://localhost:${dockerRegistryPort}`]
+            : []),
+          ...kbnServerArgs,
+        ],
+      },
+    };
+  };
+}

x-pack/test/api_integration/deployment_agnostic/default_configs/helpers.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getPreConfiguredActions } from '../../../alerting_api_integration/common/config';
+import { getTlsWebhookServerUrls } from '../../../alerting_api_integration/common/lib/get_tls_webhook_servers';
+
+export const updateKbnServerArguments = async (kbnServerArgs: string[]) => {
+  const tlsWebhookServers = await getTlsWebhookServerUrls(6300, 6399);
+  const updatedArgs = kbnServerArgs.map((arg) =>
+    arg.startsWith('--xpack.actions.preconfigured')
+      ? `--xpack.actions.preconfigured=${getPreConfiguredActions(tlsWebhookServers)}`
+      : arg
+  );
+
+  return updatedArgs;
+};