[9.3] [scout] migrate FTR logstash api tests (#262953)

.buildkite/ftr_platform_stateful_configs.yml

@@ -393,7 +393,6 @@ enabled:
   - x-pack/platform/test/api_integration/apis/grok_debugger/config.ts
   - x-pack/platform/test/api_integration/apis/file_upload/config.ts
   - x-pack/platform/test/api_integration/apis/kibana/config.ts
-  - x-pack/platform/test/api_integration/apis/logstash/config.ts
   - x-pack/platform/test/api_integration/apis/management/config.ts
   - x-pack/platform/test/api_integration/apis/management/index_management/disabled_data_enrichers/config.ts
   - x-pack/platform/test/api_integration/apis/maps/config.ts

.buildkite/scout_ci_config.yml

@@ -9,6 +9,7 @@ plugins:
     - exploratory_view
     - index_management
     - infra
+    - logstash
     - maps
     - observability
     - observability_onboarding

.github/CODEOWNERS

@@ -2941,7 +2941,6 @@ x-pack/solutions/security/plugins/security_solution/server/lib/security_integrat
 /x-pack/platform/test/functional/services/pipeline_* @elastic/logstash
 /x-pack/platform/test/functional/page_objects/logstash_page.ts @elastic/logstash
 /x-pack/platform/test/functional/apps/logstash @elastic/logstash
-/x-pack/platform/test/api_integration/apis/logstash @elastic/logstash
 #CC# /x-pack/platform/plugins/private/logstash/ @elastic/logstash
 
 # EUI team

src/platform/test/moon.yml

@@ -81,8 +81,6 @@ tags:
 fileGroups:
   src:
     - '**/*'
-    - api_integration/apis/logstash/pipeline/fixtures/*.json
-    - api_integration/apis/logstash/pipelines/fixtures/*.json
     - api_integration/apis/telemetry/fixtures/*.json
     - api_integration/apis/telemetry/fixtures/*.json
     - '!target/**/*'

src/platform/test/tsconfig.json

@@ -14,8 +14,6 @@
     "**/*",
     "../../../typings/**/*",
     "../../../src/platform/packages/shared/kbn-test/types/ftr_globals/**/*",
-    "api_integration/apis/logstash/pipeline/fixtures/*.json",
-    "api_integration/apis/logstash/pipelines/fixtures/*.json",
     "api_integration/apis/telemetry/fixtures/*.json",
     "api_integration/apis/telemetry/fixtures/*.json"
     , "../../../x-pack/platform/test/serverless/functional/test_suites/saved_objects_management/export_transform copy.ts"  ],

x-pack/platform/plugins/private/logstash/moon.yml

@@ -30,6 +30,7 @@ dependsOn:
   - '@kbn/code-editor'
   - '@kbn/react-kibana-context-render'
   - '@kbn/core-plugins-browser'
+  - '@kbn/scout'
 tags:
   - plugin
   - prod
@@ -41,6 +42,7 @@ fileGroups:
     - common/**/*
     - public/**/*
     - server/**/*
+    - test/scout/**/*
     - '!target/**/*'
   jest-config:
     - jest.config.js

x-pack/platform/plugins/private/logstash/test/scout/api/fixtures/constants.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchRoleDescriptor } from '@kbn/scout';
+
+export const API_PATHS = {
+  CLUSTER: 'api/logstash/cluster',
+  PIPELINE: (id: string) => `api/logstash/pipeline/${id}`,
+  PIPELINES: 'api/logstash/pipelines',
+  PIPELINES_DELETE: 'api/logstash/pipelines/delete',
+};
+
+export const COMMON_HEADERS = {
+  'kbn-xsrf': 'scout',
+  'x-elastic-internal-origin': 'kibana',
+};
+
+export const PIPELINE_IDS = {
+  TWEETS_AND_BEATS: 'tweets_and_beats',
+  FAST_GENERATOR: 'fast_generator',
+  /** Unique IDs for the bulk-delete test */
+  BULK_DELETE_1: 'scout_bulk_delete_1',
+  BULK_DELETE_2: 'scout_bulk_delete_2',
+};
+
+/** Expected response body for GET /api/logstash/pipeline/tweets_and_beats */
+export const EXPECTED_TWEETS_AND_BEATS_PIPELINE = {
+  id: 'tweets_and_beats',
+  description: 'ingest tweets and beats',
+  username: 'elastic',
+  pipeline:
+    'input {\n    twitter {\n        consumer_key => "enter_your_consumer_key_here"\n        consumer_secret => "enter_your_secret_here"\n        keywords => ["cloud"]\n        oauth_token => "enter_your_access_token_here"\n        oauth_token_secret => "enter_your_access_token_secret_here"\n    }\n    beats {\n        port => "5043"\n    }\n}\noutput {\n    elasticsearch {\n        hosts => ["IP Address 1:port1", "IP Address 2:port2", "IP Address 3"]\n    }\n}',
+  settings: {},
+};
+
+/** Minimum ES privileges required to manage Logstash pipelines */
+export const LOGSTASH_MANAGER_ROLE: ElasticsearchRoleDescriptor = {
+  cluster: ['manage_logstash_pipelines'],
+};
+
+/** Minimum ES privileges required to call GET /api/logstash/cluster (proxies ES info()) */
+export const LOGSTASH_CLUSTER_ROLE: ElasticsearchRoleDescriptor = {
+  cluster: ['monitor'],
+};

x-pack/platform/plugins/private/logstash/test/scout/api/fixtures/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { apiTest as base } from '@kbn/scout';
+import { getLogstashApiService, type LogstashApiService } from '../services/logstash_api_service';
+
+// Augment the shared ApiServicesFixture interface so that apiServices.logstash is typed
+// everywhere inside this plugin's Scout tests.
+declare module '@kbn/scout' {
+  interface ApiServicesFixture {
+    logstash: LogstashApiService;
+  }
+}
+
+export const apiTest = base.extend<{}, { apiServices: import('@kbn/scout').ApiServicesFixture }>({
+  apiServices: [
+    async ({ apiServices, esClient, log }, use) => {
+      await use(Object.assign(apiServices, { logstash: getLogstashApiService({ esClient, log }) }));
+    },
+    { scope: 'worker' },
+  ],
+});
+
+export * as testData from './constants';

x-pack/platform/test/api_integration/apis/logstash/cluster/index.ts → x-pack/platform/plugins/private/logstash/test/scout/api/playwright.config.ts

@@ -5,10 +5,8 @@
  * 2.0.
  */
 
-import type { FtrProviderContext } from '../../../ftr_provider_context';
+import { createPlaywrightConfig } from '@kbn/scout';
 
-export default function ({ loadTestFile }: FtrProviderContext) {
-  describe('cluster', () => {
-    loadTestFile(require.resolve('./load'));
-  });
-}
+export default createPlaywrightConfig({
+  testDir: './tests',
+});

x-pack/platform/plugins/private/logstash/test/scout/api/services/logstash_api_service.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ScoutLogger } from '@kbn/scout';
+import type { Client } from '@elastic/elasticsearch';
+
+export interface LogstashApiService {
+  /**
+   * Creates one or more minimal Logstash pipelines by ID via the ES Logstash Pipeline Management API.
+   */
+  createPipelines: (...ids: string[]) => Promise<void>;
+  /**
+   * Deletes one or more Logstash pipelines by ID using the ES Logstash Pipeline Management API.
+   * Silently ignores 404s so it is safe to call in afterAll even when setup failed partway through.
+   */
+  deletePipelines: (...ids: string[]) => Promise<void>;
+}
+
+// ES accepts an empty settings object at runtime; the TS type is overly strict
+type PipelineSettings = import('@elastic/elasticsearch').estypes.LogstashPipelineSettings;
+const EMPTY_PIPELINE_SETTINGS = {} as unknown as PipelineSettings;
+
+export const getLogstashApiService = ({
+  log,
+  esClient,
+}: {
+  log: ScoutLogger;
+  esClient: Client;
+}): LogstashApiService => {
+  return {
+    createPipelines: async (...ids: string[]) => {
+      for (const id of ids) {
+        log.debug(`[logstash] Creating pipeline '${id}'`);
+        await esClient.logstash.putPipeline({
+          id,
+          pipeline: {
+            description: `pipeline ${id}`,
+            last_modified: new Date().toISOString(),
+            pipeline: '# empty',
+            pipeline_metadata: { type: 'logstash_pipeline', version: '1' },
+            pipeline_settings: EMPTY_PIPELINE_SETTINGS,
+            username: 'elastic',
+          },
+        });
+      }
+    },
+
+    deletePipelines: async (...ids: string[]) => {
+      for (const id of ids) {
+        log.debug(`[logstash] Deleting pipeline '${id}'`);
+        await esClient.logstash.deletePipeline({ id }).catch((err) => {
+          if (err?.statusCode !== 404) {
+            log.warning(`[logstash] Failed to delete pipeline '${id}': ${err?.message}`);
+          }
+        });
+      }
+    },
+  };
+};

x-pack/platform/plugins/private/logstash/test/scout/api/tests/cluster.spec.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RoleApiCredentials } from '@kbn/scout';
+import { tags } from '@kbn/scout';
+import { expect } from '@kbn/scout/api';
+import { apiTest, testData } from '../fixtures';
+
+// The route calls client.asCurrentUser.info() which requires the ES `monitor` cluster privilege.
+// The built-in `viewer` Kibana role has no ES cluster privileges and produces a non-403 error
+// that bypasses the route's catch block, resulting in a 500. A custom role with `monitor` is
+// the minimum required privilege.
+apiTest.describe('GET /api/logstash/cluster', { tag: tags.stateful.classic }, () => {
+  let credentials: RoleApiCredentials;
+
+  apiTest.beforeAll(async ({ requestAuth }) => {
+    credentials = await requestAuth.getApiKeyForCustomRole(testData.LOGSTASH_CLUSTER_ROLE);
+  });
+
+  apiTest('should return the ES cluster info', async ({ apiClient, esClient }) => {
+    const response = await apiClient.get(testData.API_PATHS.CLUSTER, {
+      headers: { ...testData.COMMON_HEADERS, ...credentials.apiKeyHeader },
+      responseType: 'json',
+    });
+
+    expect(response).toHaveStatusCode(200);
+    expect(response.body.cluster).toBeDefined();
+
+    const esInfo = await esClient.info();
+    expect(response.body.cluster.uuid).toBe(esInfo.cluster_uuid);
+  });
+});

x-pack/platform/plugins/private/logstash/test/scout/api/tests/pipeline.spec.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RoleApiCredentials } from '@kbn/scout';
+import { tags } from '@kbn/scout';
+import { expect } from '@kbn/scout/api';
+import { apiTest, testData } from '../fixtures';
+
+apiTest.describe('/api/logstash/pipeline/{id}', { tag: tags.stateful.classic }, () => {
+  let credentials: RoleApiCredentials;
+
+  apiTest.beforeAll(async ({ requestAuth, esClient }) => {
+    credentials = await requestAuth.getApiKeyForCustomRole(testData.LOGSTASH_MANAGER_ROLE);
+
+    await esClient.logstash.putPipeline({
+      id: testData.PIPELINE_IDS.TWEETS_AND_BEATS,
+      pipeline: {
+        description: testData.EXPECTED_TWEETS_AND_BEATS_PIPELINE.description,
+        last_modified: '2017-08-02T18:59:07.724Z',
+        pipeline: testData.EXPECTED_TWEETS_AND_BEATS_PIPELINE.pipeline,
+        pipeline_metadata: { type: 'logstash_pipeline', version: '1' },
+        // ES accepts an empty settings object at runtime; the TS type is overly strict
+        pipeline_settings:
+          {} as unknown as import('@elastic/elasticsearch').estypes.LogstashPipelineSettings,
+        username: testData.EXPECTED_TWEETS_AND_BEATS_PIPELINE.username,
+      },
+    });
+  });
+
+  apiTest.afterAll(async ({ apiServices }) => {
+    await apiServices.logstash.deletePipelines(
+      testData.PIPELINE_IDS.TWEETS_AND_BEATS,
+      testData.PIPELINE_IDS.FAST_GENERATOR
+    );
+  });
+
+  apiTest('GET should return the specified pipeline', async ({ apiClient }) => {
+    const response = await apiClient.get(
+      testData.API_PATHS.PIPELINE(testData.PIPELINE_IDS.TWEETS_AND_BEATS),
+      {
+        headers: { ...testData.COMMON_HEADERS, ...credentials.apiKeyHeader },
+        responseType: 'json',
+      }
+    );
+
+    expect(response).toHaveStatusCode(200);
+    expect(response.body).toStrictEqual(testData.EXPECTED_TWEETS_AND_BEATS_PIPELINE);
+  });
+
+  apiTest('GET should return 404 for a non-existing pipeline', async ({ apiClient }) => {
+    const response = await apiClient.get(testData.API_PATHS.PIPELINE('non_existing_pipeline'), {
+      headers: { ...testData.COMMON_HEADERS, ...credentials.apiKeyHeader },
+      responseType: 'json',
+    });
+
+    expect(response).toHaveStatusCode(404);
+  });
+
+  apiTest(
+    'PUT should create the specified pipeline and DELETE should remove it',
+    async ({ apiClient }) => {
+      const createResponse = await apiClient.put(
+        testData.API_PATHS.PIPELINE(testData.PIPELINE_IDS.FAST_GENERATOR),
+        {
+          headers: { ...testData.COMMON_HEADERS, ...credentials.apiKeyHeader },
+          body: {
+            description: 'foobar baz',
+            pipeline: 'input { generator {} }\n\n output { stdout {} }',
+          },
+          responseType: 'json',
+        }
+      );
+      expect(createResponse).toHaveStatusCode(204);
+
+      const loadResponse = await apiClient.get(
+        testData.API_PATHS.PIPELINE(testData.PIPELINE_IDS.FAST_GENERATOR),
+        {
+          headers: { ...testData.COMMON_HEADERS, ...credentials.apiKeyHeader },
+          responseType: 'json',
+        }
+      );
+      expect(loadResponse).toHaveStatusCode(200);
+      expect(loadResponse.body.id).toBe(testData.PIPELINE_IDS.FAST_GENERATOR);
+      expect(loadResponse.body.description).toBe('foobar baz');
+      expect(loadResponse.body.pipeline).toBe('input { generator {} }\n\n output { stdout {} }');
+
+      const deleteResponse = await apiClient.delete(
+        testData.API_PATHS.PIPELINE(testData.PIPELINE_IDS.FAST_GENERATOR),
+        {
+          headers: { ...testData.COMMON_HEADERS, ...credentials.apiKeyHeader },
+        }
+      );
+      expect(deleteResponse).toHaveStatusCode(204);
+
+      const afterDeleteResponse = await apiClient.get(
+        testData.API_PATHS.PIPELINE(testData.PIPELINE_IDS.FAST_GENERATOR),
+        {
+          headers: { ...testData.COMMON_HEADERS, ...credentials.apiKeyHeader },
+          responseType: 'json',
+        }
+      );
+      expect(afterDeleteResponse).toHaveStatusCode(404);
+    }
+  );
+});