[8.x] [kbn-scout] Add Synthtrace as a fixture (#210505)

packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts

@@ -26,13 +26,25 @@ export class OtelSynthtraceEsClient extends SynthtraceEsClient<OtelDocument> {
     });
     this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*', 'logs-generic.otel*'];
   }
+
+  getDefaultPipeline(
+    {
+      includeSerialization,
+    }: {
+      includeSerialization?: boolean;
+    } = { includeSerialization: true }
+  ) {
+    return otelPipeline(includeSerialization);
+  }
 }
 
-function otelPipeline() {
+function otelPipeline(includeSerialization: boolean = true) {
+  const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
   return (base: Readable) => {
     return pipeline(
+      // @ts-expect-error see apm_pipeline.ts
       base,
-      getSerializeTransform(),
+      ...serializationTransform,
       getRoutingTransform(),
       getDedotTransform(),
       (err: unknown) => {

packages/kbn-scout/index.ts

@@ -16,6 +16,7 @@ export {
   createPlaywrightConfig,
   createLazyPageObject,
   ingestTestDataHook,
+  ingestSynthtraceDataHook,
 } from './src/playwright';
 export type {
   ScoutPlaywrightOptions,

packages/kbn-scout/src/common/services/synthtrace.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import {
+  ApmSynthtraceEsClient,
+  ApmSynthtraceKibanaClient,
+  InfraSynthtraceEsClient,
+  InfraSynthtraceKibanaClient,
+  LogLevel,
+  OtelSynthtraceEsClient,
+  createLogger,
+} from '@kbn/apm-synthtrace';
+import { ScoutLogger } from './logger';
+import { EsClient } from '../../types';
+
+let apmSynthtraceEsClientInstance: ApmSynthtraceEsClient | undefined;
+let infraSynthtraceEsClientInstance: InfraSynthtraceEsClient | undefined;
+let otelSynthtraceEsClientInstance: OtelSynthtraceEsClient | undefined;
+const logger = createLogger(LogLevel.info);
+
+export async function getApmSynthtraceEsClient(
+  esClient: EsClient,
+  target: string,
+  log: ScoutLogger
+) {
+  if (!apmSynthtraceEsClientInstance) {
+    const apmSynthtraceKibanaClient = new ApmSynthtraceKibanaClient({
+      logger,
+      target,
+    });
+
+    const version = await apmSynthtraceKibanaClient.fetchLatestApmPackageVersion();
+    await apmSynthtraceKibanaClient.installApmPackage(version);
+    apmSynthtraceEsClientInstance = new ApmSynthtraceEsClient({
+      client: esClient,
+      logger,
+      refreshAfterIndex: true,
+      version,
+    });
+
+    apmSynthtraceEsClientInstance.pipeline(
+      apmSynthtraceEsClientInstance.getDefaultPipeline({ includeSerialization: false })
+    );
+
+    log.serviceLoaded('apmSynthtraceClient');
+  }
+
+  return apmSynthtraceEsClientInstance;
+}
+
+export async function getInfraSynthtraceEsClient(
+  esClient: EsClient,
+  kbnUrl: string,
+  auth: { username: string; password: string },
+  log: ScoutLogger
+) {
+  if (!infraSynthtraceEsClientInstance) {
+    const infraSynthtraceKibanaClient = new InfraSynthtraceKibanaClient({
+      logger,
+      target: kbnUrl,
+      username: auth.username,
+      password: auth.password,
+    });
+
+    const version = await infraSynthtraceKibanaClient.fetchLatestSystemPackageVersion();
+    await infraSynthtraceKibanaClient.installSystemPackage(version);
+    infraSynthtraceEsClientInstance = new InfraSynthtraceEsClient({
+      client: esClient,
+      logger,
+      refreshAfterIndex: true,
+    });
+
+    infraSynthtraceEsClientInstance.pipeline(
+      infraSynthtraceEsClientInstance.getDefaultPipeline({ includeSerialization: false })
+    );
+
+    log.serviceLoaded('infraSynthtraceClient');
+  }
+
+  return infraSynthtraceEsClientInstance;
+}
+
+export function getOtelSynthtraceEsClient(esClient: EsClient, log: ScoutLogger) {
+  if (!otelSynthtraceEsClientInstance) {
+    otelSynthtraceEsClientInstance = new OtelSynthtraceEsClient({
+      client: esClient,
+      logger,
+      refreshAfterIndex: true,
+    });
+
+    otelSynthtraceEsClientInstance.pipeline(
+      otelSynthtraceEsClientInstance.getDefaultPipeline({ includeSerialization: false })
+    );
+
+    log.serviceLoaded('otelSynthtraceClient');
+  }
+
+  return otelSynthtraceEsClientInstance;
+}

packages/kbn-scout/src/playwright/fixtures/single_thread_fixtures.ts

@@ -14,6 +14,7 @@ import {
   coreWorkerFixtures,
   esArchiverFixture,
   uiSettingsFixture,
+  synthtraceFixture,
 } from './worker';
 import type {
   EsArchiverFixture,
@@ -23,23 +24,23 @@ import type {
   ScoutLogger,
   ScoutTestConfig,
   UiSettingsFixture,
+  SynthtraceFixture,
 } from './worker';
 import {
   scoutPageFixture,
   browserAuthFixture,
   pageObjectsFixture,
   validateTagsFixture,
-  BrowserAuthFixture,
-  ScoutPage,
-  PageObjects,
 } from './test';
-export type { PageObjects, ScoutPage } from './test';
+import type { BrowserAuthFixture, ScoutPage, PageObjects } from './test';
+export type { ScoutPage, PageObjects } from './test';
 
 export const scoutFixtures = mergeTests(
   // worker scope fixtures
   coreWorkerFixtures,
   esArchiverFixture,
   uiSettingsFixture,
+  synthtraceFixture,
   // api fixtures
   apiFixtures,
   // test scope fixtures
@@ -63,4 +64,7 @@ export interface ScoutWorkerFixtures extends ApiFixtures {
   esClient: EsClient;
   esArchiver: EsArchiverFixture;
   uiSettings: UiSettingsFixture;
+  apmSynthtraceEsClient: SynthtraceFixture['apmSynthtraceEsClient'];
+  infraSynthtraceEsClient: SynthtraceFixture['infraSynthtraceEsClient'];
+  otelSynthtraceEsClient: SynthtraceFixture['otelSynthtraceEsClient'];
 }

packages/kbn-scout/src/playwright/fixtures/worker/index.ts

@@ -28,3 +28,6 @@ export type { ScoutSpaceParallelFixture } from './scout_space';
 
 export { apiFixtures } from './apis';
 export type { ApiFixtures, ApiParallelWorkerFixtures } from './apis';
+
+export { synthtraceFixture } from './synthtrace';
+export type { SynthtraceFixture } from './synthtrace';

packages/kbn-scout/src/playwright/fixtures/worker/synthtrace.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Readable } from 'stream';
+import type { ApmFields, Fields, InfraDocument, OtelDocument } from '@kbn/apm-synthtrace-client';
+import Url from 'url';
+import type { SynthtraceEsClient } from '@kbn/apm-synthtrace/src/lib/shared/base_client';
+import {
+  getApmSynthtraceEsClient,
+  getInfraSynthtraceEsClient,
+  getOtelSynthtraceEsClient,
+} from '../../../common/services/synthtrace';
+import { coreWorkerFixtures } from './core_fixtures';
+import type { SynthtraceEvents } from '../../global_hooks/synthtrace_ingestion';
+
+interface SynthtraceFixtureEsClient<TFields extends Fields> {
+  index: (events: SynthtraceEvents<TFields>) => Promise<void>;
+  clean: SynthtraceEsClient<TFields>['clean'];
+}
+
+export interface SynthtraceFixture {
+  apmSynthtraceEsClient: SynthtraceFixtureEsClient<ApmFields>;
+  infraSynthtraceEsClient: SynthtraceFixtureEsClient<InfraDocument>;
+  otelSynthtraceEsClient: SynthtraceFixtureEsClient<OtelDocument>;
+}
+
+const useSynthtraceClient = async <TFields extends Fields>(
+  client: SynthtraceEsClient<TFields>,
+  use: (client: SynthtraceFixtureEsClient<TFields>) => Promise<void>
+) => {
+  const index = async (events: SynthtraceEvents<TFields>) =>
+    await client.index(Readable.from(Array.from(events).flatMap((event) => event.serialize())));
+
+  const clean = async () => await client.clean();
+
+  await use({ index, clean });
+
+  // cleanup function after all tests have ran
+  await client.clean();
+};
+
+export const synthtraceFixture = coreWorkerFixtures.extend<{}, SynthtraceFixture>({
+  apmSynthtraceEsClient: [
+    async ({ esClient, config, kbnUrl, log }, use) => {
+      const { username, password } = config.auth;
+      const kibanaUrl = new URL(kbnUrl.get());
+      const kibanaUrlWithAuth = Url.format({
+        protocol: kibanaUrl.protocol,
+        hostname: kibanaUrl.hostname,
+        port: kibanaUrl.port,
+        auth: `${username}:${password}`,
+      });
+
+      const apmSynthtraceEsClient = await getApmSynthtraceEsClient(
+        esClient,
+        kibanaUrlWithAuth,
+        log
+      );
+
+      await useSynthtraceClient<ApmFields>(apmSynthtraceEsClient, use);
+    },
+    { scope: 'worker' },
+  ],
+  infraSynthtraceEsClient: [
+    async ({ esClient, config, kbnUrl, log }, use) => {
+      const infraSynthtraceEsClient = await getInfraSynthtraceEsClient(
+        esClient,
+        kbnUrl.get(),
+        config.auth,
+        log
+      );
+
+      await useSynthtraceClient<InfraDocument>(infraSynthtraceEsClient, use);
+    },
+    { scope: 'worker' },
+  ],
+  otelSynthtraceEsClient: [
+    async ({ esClient, log }, use) => {
+      const otelSynthtraceEsClient = await getOtelSynthtraceEsClient(esClient, log);
+
+      await useSynthtraceClient<OtelDocument>(otelSynthtraceEsClient, use);
+    },
+    { scope: 'worker' },
+  ],
+});

packages/kbn-scout/src/playwright/global_hooks/index.ts

@@ -8,3 +8,4 @@
  */
 
 export { ingestTestDataHook } from './data_ingestion';
+export { ingestSynthtraceDataHook } from './synthtrace_ingestion';