[Streams] upgrade tests

.buildkite/scout_ci_config.yml

@@ -16,6 +16,7 @@ plugins:
     - security_solution
     - slo
     - spaces
+    - streams
     - streams_app
     - workflows_extensions
     - transform

src/platform/packages/shared/kbn-scout/src/playwright/page_objects/discover_app.ts

@@ -122,6 +122,11 @@ export class DiscoverApp {
     });
   }
 
+  async waitForDocViewerFlyoutOpen() {
+    const docViewer = this.page.testSubj.locator('kbnDocViewer');
+    await expect(docViewer).toBeVisible({ timeout: 30_000 });
+  }
+
   async getDocTableIndex(index: number): Promise<string> {
     const rowIndex = index - 1; // Convert to 0-based index
     const row = this.page.locator(`[data-grid-row-index="${rowIndex}"]`);

x-pack/platform/plugins/shared/streams/moon.yml

@@ -76,6 +76,8 @@ dependsOn:
   - '@kbn/actions-plugin'
   - '@kbn/apm-utils'
   - '@kbn/alerting-types'
+  - '@kbn/core-http-common'
+  - '@kbn/scout'
 tags:
   - plugin
   - prod
@@ -89,6 +91,7 @@ fileGroups:
     - server/**/*
     - public/**/*
     - types/**/*
+    - test/**/*
     - '!target/**/*'
 tasks:
   jest:

x-pack/platform/plugins/shared/streams/test/scout/api/README.md

@@ -0,0 +1,151 @@
+# Streams - Scout API Tests
+
+This directory contains Scout API tests for the Streams plugin. These tests focus on server-side API functionality without browser interaction, providing fast and reliable test coverage.
+
+## Why API Tests?
+
+API tests complement UI tests by:
+- **Running faster** - No browser overhead, tests execute in milliseconds
+- **Being more reliable** - No timing issues from UI rendering
+- **Testing edge cases** - Easier to test error scenarios and validation
+- **Reducing flakiness** - Deterministic API responses vs. async UI updates
+
+## Directory Structure
+
+```
+api/
+├── playwright.config.ts          # Scout API test configuration
+├── constants.ts                  # Common headers and constants
+├── fixtures/
+│   ├── index.ts                 # Test fixtures extending apiTest
+│   └── constants.ts             # User roles and API headers
+├── services/
+│   └── streams_api_service.ts   # Streams API helper service
+└── tests/
+    ├── global.setup.ts          # Global setup (enables Streams)
+    ├── processing_simulate.spec.ts
+    ├── routing_fork_stream.spec.ts
+    ├── schema_field_mapping.spec.ts
+    └── lifecycle_retention.spec.ts
+```
+
+## Running Tests
+
+### Prerequisites
+
+1. Start Elasticsearch and Kibana servers:
+   ```bash
+   node scripts/scout run-servers --stateful
+   ```
+
+2. Run the API tests:
+   ```bash
+   npx playwright test --config x-pack/platform/plugins/shared/streams/test/scout/api/playwright.config.ts
+   ```
+
+### Running Specific Tests
+
+```bash
+# Run only routing tests
+npx playwright test --config x-pack/platform/plugins/shared/streams/test/scout/api/playwright.config.ts -g "routing"
+
+# Run only processing tests
+npx playwright test --config x-pack/platform/plugins/shared/streams/test/scout/api/playwright.config.ts -g "processing"
+```
+
+## Test Coverage
+
+### Processing Tests (`processing/simulate_processing.spec.ts`)
+- Grok pattern simulation
+- Dissect pattern simulation
+- Multiple processing steps
+- Invalid pattern handling
+- Error scenarios
+
+### Routing Tests (`routing/fork_stream.spec.ts`)
+- Create child streams via fork API
+- Disabled routing rules
+- Nested child streams
+- Delete streams
+- Complex conditions (AND/OR)
+- Duplicate stream names
+- Invalid conditions
+
+### Schema Tests (`schema/field_mapping.spec.ts`)
+- Get unmapped fields
+- Simulate field mappings with various types:
+  - keyword, long, boolean, double, ip, date, geo_point
+- Invalid field types
+- Nested field names
+- Multiple field definitions
+
+### Lifecycle Tests (`lifecycle/retention.spec.ts`)
+- Get lifecycle stats
+- Get lifecycle explain
+- Update retention settings
+- Inheritance from parent
+- Different time units (hours, days)
+- Clear custom retention
+
+## Writing New Tests
+
+1. **Import the test fixture:**
+   ```typescript
+   import { streamsApiTest as apiTest } from '../../fixtures';
+   import { COMMON_API_HEADERS } from '../../fixtures/constants';
+   ```
+
+2. **Set up authentication in beforeAll:**
+   ```typescript
+   apiTest.beforeAll(async ({ samlAuth }) => {
+     const credentials = await samlAuth.asStreamsAdmin();
+     adminCookieHeader = credentials.cookieHeader;
+   });
+   ```
+
+3. **Use apiClient for HTTP requests:**
+   ```typescript
+   apiTest('should do something', async ({ apiClient }) => {
+     const { statusCode, body } = await apiClient.post('api/streams/...', {
+       headers: { ...COMMON_API_HEADERS, ...adminCookieHeader },
+       body: { ... },
+       responseType: 'json',
+     });
+     expect(statusCode).toBe(200);
+   });
+   ```
+
+4. **Clean up test data in afterEach:**
+   ```typescript
+   apiTest.afterEach(async ({ apiServices }) => {
+     await apiServices.streamsTest.cleanupTestStreams('logs.my-prefix');
+   });
+   ```
+
+## Migration from UI Tests
+
+When moving functionality from UI tests to API tests:
+
+1. **Keep UI tests for:**
+   - Critical user journeys
+   - Visual/UX verification
+   - Component interactions
+
+2. **Move to API tests:**
+   - CRUD operations
+   - Data validation
+   - Error handling
+   - Edge cases
+   - Business logic verification
+
+## Debugging
+
+Enable verbose logging:
+```bash
+DEBUG=scout:* npx playwright test --config ...
+```
+
+View test artifacts:
+```bash
+ls -la x-pack/platform/plugins/shared/streams/test/scout/api/.scout/
+```

x-pack/platform/plugins/shared/streams/test/scout/api/constants.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
+
+export const COMMON_HEADERS = {
+  'kbn-xsrf': 'some-xsrf-token',
+  'x-elastic-internal-origin': 'kibana',
+  [ELASTIC_HTTP_VERSION_HEADER]: '1',
+} as const;

x-pack/platform/plugins/shared/streams/test/scout/api/fixtures/constants.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { KibanaRole, ScoutTestConfig } from '@kbn/scout';
+
+// Headers for internal APIs (version 1)
+export const COMMON_API_HEADERS = {
+  'kbn-xsrf': 'some-xsrf-token',
+  'x-elastic-internal-origin': 'kibana',
+  'elastic-api-version': '1',
+} as const;
+
+// Headers for public APIs (version 2023-10-31)
+export const PUBLIC_API_HEADERS = {
+  'kbn-xsrf': 'some-xsrf-token',
+  'x-elastic-internal-origin': 'kibana',
+  'elastic-api-version': '2023-10-31',
+} as const;
+
+/**
+ * Returns streams user roles with privileges appropriate for the deployment type.
+ * Some cluster privileges (manage_ilm, manage_data_stream_global_retention) are not
+ * supported in serverless mode.
+ */
+export function getStreamsUsers(config: ScoutTestConfig): Record<string, KibanaRole> {
+  const isServerless = config.serverless;
+
+  // Cluster privileges that are only available in stateful deployments
+  const statefulOnlyClusterPrivileges = isServerless
+    ? []
+    : ['manage_ilm', 'manage_data_stream_global_retention'];
+
+  return {
+    streamsAdmin: {
+      kibana: [
+        {
+          base: ['all'],
+          feature: {},
+          spaces: ['*'],
+        },
+      ],
+      elasticsearch: {
+        cluster: [
+          'manage_index_templates',
+          'monitor',
+          'manage_pipeline',
+          ...statefulOnlyClusterPrivileges,
+        ],
+        indices: [
+          { names: ['logs*'], privileges: ['all'] },
+          { names: ['.ds-logs*'], privileges: ['all'] },
+          { names: ['.streams*'], privileges: ['all'] },
+          { names: ['.kibana_streams*'], privileges: ['all'] },
+        ],
+      },
+    },
+
+    streamsReadOnly: {
+      kibana: [
+        {
+          base: ['read'],
+          feature: {},
+          spaces: ['*'],
+        },
+      ],
+      elasticsearch: {
+        cluster: ['monitor'],
+        indices: [
+          { names: ['logs*'], privileges: ['read', 'view_index_metadata'] },
+          { names: ['.ds-logs*'], privileges: ['read', 'view_index_metadata'] },
+          { names: ['.kibana_streams*'], privileges: ['read', 'view_index_metadata'] },
+        ],
+      },
+    },
+
+    streamsUnauthorized: {
+      kibana: [
+        {
+          base: [],
+          feature: {},
+          spaces: ['*'],
+        },
+      ],
+      elasticsearch: {
+        cluster: [],
+        indices: [],
+      },
+    },
+  };
+}

x-pack/platform/plugins/shared/streams/test/scout/api/fixtures/index.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { apiTest } from '@kbn/scout';
+import type {
+  RoleApiCredentials,
+  RoleSessionCredentials,
+  ApiServicesFixture,
+  RequestAuthFixture,
+  SamlAuth,
+} from '@kbn/scout';
+import type { StreamsTestApiService } from '../services/streams_api_service';
+import { getStreamsTestApiService } from '../services/streams_api_service';
+import { getStreamsUsers } from './constants';
+
+export interface StreamsSamlAuthFixture extends SamlAuth {
+  asStreamsAdmin: () => Promise<RoleSessionCredentials>;
+  asStreamsReadOnly: () => Promise<RoleSessionCredentials>;
+  asStreamsUnauthorized: () => Promise<RoleSessionCredentials>;
+}
+
+export interface StreamsRequestAuthFixture extends RequestAuthFixture {
+  loginAsStreamsAdmin: () => Promise<RoleApiCredentials>;
+  loginAsStreamsReadOnly: () => Promise<RoleApiCredentials>;
+}
+
+export interface StreamsApiServicesFixture extends ApiServicesFixture {
+  streamsTest: StreamsTestApiService;
+}
+
+export const streamsApiTest = apiTest.extend<{
+  requestAuth: StreamsRequestAuthFixture;
+  samlAuth: StreamsSamlAuthFixture;
+  apiServices: StreamsApiServicesFixture;
+}>({
+  requestAuth: async ({ requestAuth, config }, use) => {
+    const streamsUsers = getStreamsUsers(config);
+
+    const loginAsStreamsAdmin = async () =>
+      requestAuth.getApiKeyForCustomRole(streamsUsers.streamsAdmin);
+
+    const loginAsStreamsReadOnly = async () =>
+      requestAuth.getApiKeyForCustomRole(streamsUsers.streamsReadOnly);
+
+    const extendedRequestAuth: StreamsRequestAuthFixture = {
+      ...requestAuth,
+      loginAsStreamsAdmin,
+      loginAsStreamsReadOnly,
+    };
+    await use(extendedRequestAuth);
+  },
+
+  samlAuth: async ({ samlAuth, config }, use) => {
+    const streamsUsers = getStreamsUsers(config);
+
+    const asStreamsAdmin = async () => samlAuth.asInteractiveUser(streamsUsers.streamsAdmin);
+
+    const asStreamsReadOnly = async () => samlAuth.asInteractiveUser(streamsUsers.streamsReadOnly);
+
+    const asStreamsUnauthorized = async () =>
+      samlAuth.asInteractiveUser(streamsUsers.streamsUnauthorized);
+
+    const extendedSamlAuth: StreamsSamlAuthFixture = {
+      ...samlAuth,
+      asStreamsAdmin,
+      asStreamsReadOnly,
+      asStreamsUnauthorized,
+    };
+
+    await use(extendedSamlAuth);
+  },
+
+  apiServices: async ({ apiServices, kbnClient, log }, use) => {
+    const extendedApiServices = apiServices as StreamsApiServicesFixture;
+    extendedApiServices.streamsTest = getStreamsTestApiService({ kbnClient, log });
+    await use(extendedApiServices);
+  },
+});
+
+export { getStreamsUsers } from './constants';
+export { COMMON_API_HEADERS, PUBLIC_API_HEADERS } from './constants';

x-pack/platform/plugins/shared/streams/test/scout/api/playwright.config.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createPlaywrightConfig } from '@kbn/scout';
+
+export default createPlaywrightConfig({
+  testDir: './tests',
+  runGlobalSetup: true,
+});