[Dataset quality] configuring dockerized package registry in tests

src/platform/packages/private/kbn-journeys/journey/journey_ftr_config.ts

@@ -7,22 +7,20 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import Path from 'path';
-
-import { v4 as uuidV4 } from 'uuid';
 import { REPO_ROOT } from '@kbn/repo-info';
 import {
-  type FtrConfigProviderContext,
   type FtrConfigProvider,
+  type FtrConfigProviderContext,
   defineDockerServersConfig,
-  fleetPackageRegistryDockerImage,
+  packageRegistryDocker,
 } from '@kbn/test';
-import path from 'path';
 import apm from 'elastic-apm-node';
+import path from 'path';
+import { v4 as uuidV4 } from 'uuid';
 import { services } from '../services';
 import type { AnyStep } from './journey';
-import type { JourneyConfig } from './journey_config';
 import { JOURNEY_APM_CONFIG } from './journey_apm_config';
+import type { JourneyConfig } from './journey_config';
 
 export function makeFtrConfigProvider(
   config: JourneyConfig<any>,
@@ -38,7 +36,7 @@ export function makeFtrConfigProvider(
       ? 'x-pack/platform/test/functional/config.base.ts'
       : 'src/platform/test/functional/config.base.js';
     const ftrConfigPath = configPath ?? defaultConfigPath;
-    const baseConfig = (await readConfigFile(Path.resolve(REPO_ROOT, ftrConfigPath))).getAll();
+    const baseConfig = (await readConfigFile(path.resolve(REPO_ROOT, ftrConfigPath))).getAll();
 
     const testBuildId = process.env.BUILDKITE_BUILD_ID ?? `local-${uuidV4()}`;
     const testJobId = process.env.BUILDKITE_JOB_ID ?? `local-${uuidV4()}`;
@@ -84,30 +82,11 @@ export function makeFtrConfigProvider(
       .flatMap(([key, value]) => (value === null || value === undefined ? [] : `${key}=${value}`))
       .join(',');
 
-    /**
-     * This is used by CI to set the docker registry port
-     * you can also define this environment variable locally when running tests which
-     * will spin up a local docker package registry locally for you
-     * if this is defined it takes precedence over the `packageRegistryOverride` variable
-     */
-    const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
-
-    const packageRegistryConfig = path.join(__dirname, '../fixtures/package_registry_config.yml');
-    const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
-
     return {
       ...baseConfig,
 
       dockerServers: defineDockerServersConfig({
-        registry: {
-          enabled: !!dockerRegistryPort,
-          image: fleetPackageRegistryDockerImage,
-          portInContainer: 8080,
-          port: dockerRegistryPort,
-          args: dockerArgs,
-          waitForLogLine: 'package manifests loaded',
-          waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
-        },
+        registry: packageRegistryDocker,
       }),
 
       mochaOpts: {

src/platform/packages/shared/kbn-scout/src/config/serverless/serverless.base.config.ts

@@ -7,31 +7,19 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { resolve, join } from 'path';
-import { format as formatUrl } from 'url';
-import Fs from 'fs';
-
 import { CA_CERT_PATH, kibanaDevServiceAccount } from '@kbn/dev-utils';
+import { MOCK_IDP_REALM_NAME } from '@kbn/mock-idp-utils';
+import { REPO_ROOT } from '@kbn/repo-info';
 import {
-  fleetPackageRegistryDockerImage,
   defineDockerServersConfig,
   getDockerFileMountPath,
+  packageRegistryDocker,
 } from '@kbn/test';
-import { MOCK_IDP_REALM_NAME } from '@kbn/mock-idp-utils';
-import { REPO_ROOT } from '@kbn/repo-info';
+import Fs from 'fs';
+import { resolve } from 'path';
+import { format as formatUrl } from 'url';
 import type { ScoutServerConfig } from '../../types';
-import { SAML_IDP_PLUGIN_PATH, SERVERLESS_IDP_METADATA_PATH, JWKS_PATH } from '../constants';
-
-const packageRegistryConfig = join(__dirname, './package_registry_config.yml');
-const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
-
-/**
- * This is used by CI to set the docker registry port
- * you can also define this environment variable locally when running tests which
- * will spin up a local docker package registry locally for you
- * if this is defined it takes precedence over the `packageRegistryOverride` variable
- */
-const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
+import { JWKS_PATH, SAML_IDP_PLUGIN_PATH, SERVERLESS_IDP_METADATA_PATH } from '../constants';
 
 const servers = {
   elasticsearch: {
@@ -55,15 +43,7 @@ export const defaultConfig: ScoutServerConfig = {
   serverless: true,
   servers,
   dockerServers: defineDockerServersConfig({
-    registry: {
-      enabled: !!dockerRegistryPort,
-      image: fleetPackageRegistryDockerImage,
-      portInContainer: 8080,
-      port: dockerRegistryPort,
-      args: dockerArgs,
-      waitForLogLine: 'package manifests loaded',
-      waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
-    },
+    registry: packageRegistryDocker,
   }),
   esTestCluster: {
     from: 'serverless',
@@ -72,7 +52,6 @@ export const defaultConfig: ScoutServerConfig = {
       'xpack.security.authc.realms.file.file1.order=-100',
       `xpack.security.authc.realms.native.native1.enabled=false`,
       `xpack.security.authc.realms.native.native1.order=-97`,
-
       'xpack.security.authc.realms.jwt.jwt1.allowed_audiences=elasticsearch',
       `xpack.security.authc.realms.jwt.jwt1.allowed_issuer=https://kibana.elastic.co/jwt/`,
       `xpack.security.authc.realms.jwt.jwt1.allowed_signature_algorithms=[RS256]`,

src/platform/packages/shared/kbn-scout/src/config/stateful/base.config.ts

@@ -20,21 +20,10 @@ import {
   MOCK_IDP_REALM_NAME,
 } from '@kbn/mock-idp-utils';
 import { REPO_ROOT } from '@kbn/repo-info';
-import { defineDockerServersConfig, fleetPackageRegistryDockerImage } from '@kbn/test';
+import { defineDockerServersConfig, packageRegistryDocker } from '@kbn/test';
 import type { ScoutServerConfig } from '../../types';
 import { SAML_IDP_PLUGIN_PATH, STATEFUL_IDP_METADATA_PATH } from '../constants';
 
-const packageRegistryConfig = join(__dirname, './package_registry_config.yml');
-const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
-
-/**
- * This is used by CI to set the docker registry port
- * you can also define this environment variable locally when running tests which
- * will spin up a local docker package registry locally for you
- * if this is defined it takes precedence over the `packageRegistryOverride` variable
- */
-const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
-
 // if config is executed on CI or locally
 const isRunOnCI = process.env.CI;
 
@@ -60,15 +49,7 @@ const kbnUrl = `${servers.kibana.protocol}://${servers.kibana.hostname}:${server
 export const defaultConfig: ScoutServerConfig = {
   servers,
   dockerServers: defineDockerServersConfig({
-    registry: {
-      enabled: !!dockerRegistryPort,
-      image: fleetPackageRegistryDockerImage,
-      portInContainer: 8080,
-      port: dockerRegistryPort,
-      args: dockerArgs,
-      waitForLogLine: 'package manifests loaded',
-      waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
-    },
+    registry: packageRegistryDocker,
   }),
   esTestCluster: {
     from: 'snapshot',

src/platform/packages/shared/kbn-test/index.ts

@@ -81,9 +81,8 @@ export * from './src/find_test_plugin_paths';
 
 export { getDockerFileMountPath } from '@kbn/es';
 
-// Docker image to use for Fleet API integration tests.
-// This image comes from the latest successful build of https://buildkite.com/elastic/kibana-package-registry-promote
-// which is promoted after acceptance tests succeed against docker.elastic.co/package-registry/distribution:lite
-export const fleetPackageRegistryDockerImage =
-  process.env.FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE ||
-  'docker.elastic.co/kibana-ci/package-registry-distribution:lite';
+export {
+  fleetPackageRegistryDockerImage,
+  packageRegistryDocker,
+  dockerRegistryPort,
+} from './src/functional_test_runner';

src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/docker_servers/index.ts

@@ -9,3 +9,4 @@
 
 export * from './docker_servers_service';
 export * from './define_docker_servers_config';
+export * from './package_registry';

src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/docker_servers/package_registry/index.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { join } from 'path';
+
+// Docker image to use for Fleet API integration tests.
+// This image comes from the latest successful build of https://buildkite.com/elastic/kibana-package-registry-promote
+// which is promoted after acceptance tests succeed against docker.elastic.co/package-registry/distribution:lite
+export const fleetPackageRegistryDockerImage =
+  process.env.FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE ||
+  'docker.elastic.co/kibana-ci/package-registry-distribution:lite';
+
+const packageRegistryConfig = join(__dirname, './package_registry_config.yml');
+const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
+
+/**
+ * This is used by CI to set the docker registry port
+ * you can also define this environment variable locally when running tests which
+ * will spin up a local docker package registry locally for you
+ * if this is defined it takes precedence over the `packageRegistryOverride` variable
+ */
+export const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
+
+export const packageRegistryDocker = {
+  enabled: !!dockerRegistryPort,
+  image: fleetPackageRegistryDockerImage,
+  portInContainer: 8080,
+  port: dockerRegistryPort,
+  args: dockerArgs,
+  waitForLogLine: 'package manifests loaded',
+  waitForLogLineTimeoutMs: 60 * 4 * 1000, // 4 minutes
+};

x-pack/platform/plugins/shared/dataset_quality/README.md

@@ -78,7 +78,7 @@ node x-pack/platform/plugins/shared/dataset_quality/scripts/api --runner --grep-
 
 For tests using package registry we have enabled a configuration that uses a dockerized lite version to execute the tests in the CI, this will reduce the flakyness of them when calling the real endpoint.
 
-To be able to run this version locally you must have a docker daemon running in your systema and set `FLEET_PACKAGE_REGISTRY_PORT` env var. In order to set this variable execute
+To be able to run this version locally you must have a docker daemon running in your system and set `FLEET_PACKAGE_REGISTRY_PORT` env var. In order to set this variable execute
 
 ```
 export set FLEET_PACKAGE_REGISTRY_PORT=12345

x-pack/platform/test/api_integration_deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts

@@ -4,12 +4,10 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import type { FtrConfigProviderContext, Config } from '@kbn/test';
-import { fleetPackageRegistryDockerImage, defineDockerServersConfig } from '@kbn/test';
-
-import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
+import type { Config, FtrConfigProviderContext } from '@kbn/test';
+import { defineDockerServersConfig, dockerRegistryPort, packageRegistryDocker } from '@kbn/test';
 import type { ServerlessProjectType } from '@kbn/es';
-import path from 'path';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { DeploymentAgnosticCommonServices } from '../services';
 import { services } from '../services';
 import { LOCAL_PRODUCT_DOC_PATH } from './common_paths';
@@ -58,22 +56,12 @@ export function createServerlessFeatureFlagTestConfig<T extends DeploymentAgnost
   options: CreateTestConfigOptions<T>
 ) {
   return async ({ readConfigFile }: FtrConfigProviderContext): Promise<Config> => {
-    const packageRegistryConfig = path.join(__dirname, './fixtures/package_registry_config.yml');
-    const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
     let kbnServerArgs: string[] = [];
 
     if (options.kbnServerArgs) {
       kbnServerArgs = await updateKbnServerArguments(options.kbnServerArgs);
     }
 
-    /**
-     * This is used by CI to set the docker registry port
-     * you can also define this environment variable locally when running tests which
-     * will spin up a local docker package registry locally for you
-     * if this is defined it takes precedence over the `packageRegistryOverride` variable
-     */
-    const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
-
     const svlSharedConfig = await readConfigFile(
       require.resolve('../../serverless/shared/config.base.ts')
     );
@@ -87,15 +75,7 @@ export function createServerlessFeatureFlagTestConfig<T extends DeploymentAgnost
         ...(options.services || services),
       },
       dockerServers: defineDockerServersConfig({
-        registry: {
-          enabled: !!dockerRegistryPort,
-          image: fleetPackageRegistryDockerImage,
-          portInContainer: 8080,
-          port: dockerRegistryPort,
-          args: dockerArgs,
-          waitForLogLine: 'package manifests loaded',
-          waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
-        },
+        registry: packageRegistryDocker,
       }),
       esTestCluster: {
         ...svlSharedConfig.get('esTestCluster'),

x-pack/platform/test/api_integration_deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts

@@ -4,26 +4,27 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import { STATEFUL_ROLES_ROOT_PATH } from '@kbn/es';
 import {
-  MOCK_IDP_REALM_NAME,
-  MOCK_IDP_ENTITY_ID,
-  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
-  MOCK_IDP_ATTRIBUTE_ROLES,
   MOCK_IDP_ATTRIBUTE_EMAIL,
   MOCK_IDP_ATTRIBUTE_NAME,
+  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
+  MOCK_IDP_ATTRIBUTE_ROLES,
+  MOCK_IDP_ENTITY_ID,
+  MOCK_IDP_REALM_NAME,
 } from '@kbn/mock-idp-utils';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import type { FtrConfigProviderContext } from '@kbn/test';
 import {
-  fleetPackageRegistryDockerImage,
+  defineDockerServersConfig,
+  dockerRegistryPort,
   esTestConfig,
   kbnTestConfig,
+  packageRegistryDocker,
   systemIndicesSuperuser,
-  defineDockerServersConfig,
 } from '@kbn/test';
-import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
 import path from 'path';
-import { REPO_ROOT } from '@kbn/repo-info';
-import { STATEFUL_ROLES_ROOT_PATH } from '@kbn/es';
 import type { DeploymentAgnosticCommonServices } from '../services';
 import { services } from '../services';
 import { AI_ASSISTANT_SNAPSHOT_REPO_PATH, LOCAL_PRODUCT_DOC_PATH } from './common_paths';
@@ -45,22 +46,12 @@ export function createStatefulFeatureFlagTestConfig<T extends DeploymentAgnostic
     // if config is executed on CI or locally
     const isRunOnCI = process.env.CI;
 
-    const packageRegistryConfig = path.join(__dirname, './fixtures/package_registry_config.yml');
-    const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`];
     let kbnServerArgs: string[] = [];
 
     if (options.kbnServerArgs) {
       kbnServerArgs = await updateKbnServerArguments(options.kbnServerArgs);
     }
 
-    /**
-     * This is used by CI to set the docker registry port
-     * you can also define this environment variable locally when running tests which
-     * will spin up a local docker package registry locally for you
-     * if this is defined it takes precedence over the `packageRegistryOverride` variable
-     */
-    const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT;
-
     const xPackAPITestsConfig = await readConfigFile(
       require.resolve('../../api_integration/config.ts')
     );
@@ -91,15 +82,7 @@ export function createStatefulFeatureFlagTestConfig<T extends DeploymentAgnostic
       servers,
       testConfigCategory: ScoutTestRunConfigCategory.API_TEST,
       dockerServers: defineDockerServersConfig({
-        registry: {
-          enabled: !!dockerRegistryPort,
-          image: fleetPackageRegistryDockerImage,
-          portInContainer: 8080,
-          port: dockerRegistryPort,
-          args: dockerArgs,
-          waitForLogLine: 'package manifests loaded',
-          waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
-        },
+        registry: packageRegistryDocker,
       }),
       testFiles: options.testFiles,
       security: { disableTestUser: true },