[Space time] extending Scout with perfTracker fixture

package.json

@@ -1797,6 +1797,7 @@
     "json5": "^2.2.3",
     "jsondiffpatch": "0.4.1",
     "license-checker": "^25.0.1",
+    "lighthouse": "^12.4.0",
     "listr2": "^8.2.5",
     "lmdb": "^2.9.2",
     "marge": "^1.0.1",

renovate.json

@@ -160,6 +160,7 @@
         "@types/supertest",
         "@types/react-test-renderer",
         "babel-plugin-istanbul",
+        "lighthouse",
         "nyc",
         "oboe",
         "pixelmatch",

src/platform/packages/shared/kbn-scout/index.ts

@@ -12,6 +12,7 @@ export {
   expect,
   test,
   spaceTest,
+  lighthouseTest,
   globalSetupHook,
   tags,
   createPlaywrightConfig,
@@ -38,6 +39,6 @@ export type {
 } from './src/types';
 
 // re-export from Playwright
-export type { Locator } from 'playwright/test';
+export type { Locator, CDPSession } from 'playwright/test';
 
 export { measurePerformance, measurePerformanceAsync } from './src/common';

src/platform/packages/shared/kbn-scout/src/config/utils/save_scout_test_config.test.ts

@@ -11,6 +11,7 @@ import path from 'path';
 import Fs from 'fs';
 import { ToolingLog } from '@kbn/tooling-log';
 import { saveScoutTestConfigOnDisk } from './save_scout_test_config';
+import { ServerlessProjectType } from '@kbn/es';
 
 const MOCKED_SCOUT_SERVERS_ROOT = '/mock/repo/root/scout/servers';
 
@@ -34,6 +35,7 @@ const testServersConfig = {
     password: 'changeme',
   },
   serverless: true,
+  projectType: 'oblt' as ServerlessProjectType,
   isCloud: true,
   license: 'trial',
   cloudUsersFilePath: '/path/to/users',

src/platform/packages/shared/kbn-scout/src/playwright/config/create_config.test.ts

@@ -49,6 +49,8 @@ describe('createPlaywrightConfig', () => {
     expect(config.use).toEqual({
       serversConfigDir: SCOUT_SERVERS_ROOT,
       [VALID_CONFIG_MARKER]: true,
+      actionTimeout: 10000,
+      navigationTimeout: 20000,
       screenshot: 'only-on-failure',
       testIdAttribute: 'data-test-subj',
       trace: 'on-first-retry',

src/platform/packages/shared/kbn-scout/src/playwright/config/create_config.ts

@@ -72,6 +72,8 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri
     ],
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
+      actionTimeout: 10000, // Shorten timeout for actions like `click()`
+      navigationTimeout: 20000, // Shorter timeout for page navigations
       // 'configName' is not defined by default to enforce using '--project' flag when running the tests
       testIdAttribute: 'data-test-subj',
       serversConfigDir: SCOUT_SERVERS_ROOT,

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/single_thread_fixtures.ts

@@ -15,6 +15,7 @@ import {
   esArchiverFixture,
   uiSettingsFixture,
   synthtraceFixture,
+  lighthouseFixture,
 } from './worker';
 import type {
   EsArchiverFixture,
@@ -31,9 +32,12 @@ import {
   browserAuthFixture,
   pageObjectsFixture,
   validateTagsFixture,
+  persistentContext,
+  perfTrackerFixture,
 } from './test';
-import type { BrowserAuthFixture, ScoutPage, PageObjects } from './test';
+import type { BrowserAuthFixture, ScoutPage, PageObjects, PerfTrackerFixture } from './test';
 export type { ScoutPage, PageObjects } from './test';
+export type { LighthouseAuditOptions } from './worker';
 
 export const scoutFixtures = mergeTests(
   // worker scope fixtures
@@ -47,13 +51,16 @@ export const scoutFixtures = mergeTests(
   browserAuthFixture,
   scoutPageFixture,
   pageObjectsFixture,
-  validateTagsFixture
+  validateTagsFixture,
+  // performance fixtures
+  perfTrackerFixture
 );
 
 export interface ScoutTestFixtures {
   browserAuth: BrowserAuthFixture;
   page: ScoutPage;
   pageObjects: PageObjects;
+  perfTracker: PerfTrackerFixture;
 }
 
 export interface ScoutWorkerFixtures extends ApiFixtures {
@@ -68,3 +75,5 @@ export interface ScoutWorkerFixtures extends ApiFixtures {
   infraSynthtraceEsClient: SynthtraceFixture['infraSynthtraceEsClient'];
   otelSynthtraceEsClient: SynthtraceFixture['otelSynthtraceEsClient'];
 }
+
+export const lighthouseFixtures = mergeTests(scoutFixtures, persistentContext, lighthouseFixture);

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/test/context/index.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import * as os from 'os';
+import getPort from 'get-port';
+import { BrowserContext, chromium } from 'playwright/test';
+import { coreWorkerFixtures } from '../../worker';
+
+/**
+ * Launches browser with persistent context across multiple tests / browser windows in the same test.
+ * E.g. Lighthouse launches a new browser window and the authentication state
+ * is not persisted between windows by default, so we can't do page audit without persistent context.
+ */
+export const persistentContext = coreWorkerFixtures.extend<
+  {
+    context: BrowserContext;
+  },
+  { debuggingPort: number }
+>({
+  debuggingPort: [
+    async ({ log }, use) => {
+      const port = await getPort({ port: [9222, 9223, 9224] });
+      log.serviceLoaded(`remote debugging port [${port}]`);
+      use(port);
+    },
+    { scope: 'worker' },
+  ],
+  context: [
+    async ({ log, debuggingPort }, use) => {
+      const userDataDir = os.tmpdir();
+      const context = await chromium.launchPersistentContext(userDataDir, {
+        args: [`--remote-debugging-port=${debuggingPort}`],
+      });
+      log.serviceLoaded(`persistentContext on port [${debuggingPort}]`);
+      await use(context);
+      await context.close();
+    },
+    { scope: 'test' },
+  ],
+});

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/test/index.ts

@@ -14,3 +14,6 @@ export type { ScoutPage } from './scout_page';
 export { validateTagsFixture } from './validate_tags';
 export { pageObjectsFixture, pageObjectsParallelFixture } from './page_objects';
 export type { PageObjects } from './page_objects';
+export { persistentContext } from './context';
+export { perfTrackerFixture } from './performance';
+export type { PerfTrackerFixture } from './performance';

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/test/page_objects/parallel.ts

@@ -8,7 +8,7 @@
  */
 
 import { PageObjects, createCorePageObjects } from '../../../page_objects';
-import { ScoutSpaceParallelFixture } from '../../worker';
+import { ScoutSpaceParallelFixture, ScoutTestConfig } from '../../worker';
 import { scoutPageParallelFixture } from '../scout_page';
 
 /**
@@ -23,10 +23,10 @@ export const pageObjectsParallelFixture = scoutPageParallelFixture.extend<
   {
     pageObjects: PageObjects;
   },
-  { scoutSpace: ScoutSpaceParallelFixture }
+  { scoutSpace: ScoutSpaceParallelFixture; config: ScoutTestConfig }
 >({
-  pageObjects: async ({ page, log }, use) => {
-    const corePageObjects = createCorePageObjects(page);
+  pageObjects: async ({ page, config, log }, use) => {
+    const corePageObjects = createCorePageObjects({ page, config, log });
     log.serviceLoaded(`pageObjects`);
     await use(corePageObjects);
   },

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/test/page_objects/single_thread.ts

@@ -8,6 +8,7 @@
  */
 
 import { PageObjects, createCorePageObjects } from '../../../page_objects';
+import { ScoutTestConfig } from '../../worker';
 import { scoutPageFixture } from '../scout_page';
 
 /**
@@ -18,11 +19,14 @@ import { scoutPageFixture } from '../scout_page';
  *
  * Note: Page Objects are lazily instantiated on first access.
  */
-export const pageObjectsFixture = scoutPageFixture.extend<{
-  pageObjects: PageObjects;
-}>({
-  pageObjects: async ({ page, log }, use) => {
-    const corePageObjects = createCorePageObjects(page);
+export const pageObjectsFixture = scoutPageFixture.extend<
+  {
+    pageObjects: PageObjects;
+  },
+  { config: ScoutTestConfig }
+>({
+  pageObjects: async ({ page, log, config }, use) => {
+    const corePageObjects = createCorePageObjects({ page, config, log });
     log.serviceLoaded('pageObjects');
     await use(corePageObjects);
   },

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/test/performance/README.md

@@ -0,0 +1,126 @@
+## Performance Tracker Fixture
+
+### Overview
+
+`perfTrackerFixture` is a Playwright fixture designed to analyze JavaScript bundle performance and page-level performance metrics in Kibana by leveraging Chrome DevTools Protocol (CDP). It intercepts network requests, filters static bundles, and computes bundle size statistics per page load. Additionally, it collects CDP Performance Domain Metrics, allowing in-depth analysis of rendering and script execution times.
+
+### Key Features
+
+- Uses CDP session to monitor network requests in Playwright tests.
+- Exposes `waitForJsLoad` to ensure all incoming bundle requests are fully resolved before proceeding.
+- Exposes `captureBundleResponses` to start tracking network requests and collect JavaScript bundle responses.
+- Exposes `collectJsBundleStats` to aggregate all captured responses by plugin, making validation easier in tests.
+- Automatically attaches collected JS bundle stats as a JSON artifact in test reports, making them available in the `Playwright HTML report`.
+- Captures `CDP Performance Domain Metrics`, including:
+  - JavaScript Heap Usage (jsHeapUsedSize, jsHeapTotalSize)
+  - CPU Execution Time (cpuTime)
+  - Script Execution Time (scriptTime)
+  - Layout & Rendering Performance (layoutTime, layoutCount, styleRecalcCount)
+  - Frames Per Second (FPS) (fps)
+  - DOM Complexity Metrics (nodesCount, documentsCount)
+
+
+### Usage: capturing JS bundles on page
+
+```ts
+test.describe(
+  'Discover App - Performance Metrics & Bundle Analysis',
+  { tag: [...tags.DEPLOYMENT_AGNOSTIC, ...tags.PERFORMANCE] },
+  () => {
+    let cdp: CDPSession;
+
+    test.beforeEach(async ({ browserAuth, page, context, perfTracker }) => {
+      await browserAuth.loginAsAdmin();
+      cdp = await context.newCDPSession(page);
+      await cdp.send('Network.enable');
+      // load the starting page, e.g. '/app/home'
+      await perfTracker.waitForJsLoad(cdp); // Ensure JS bundles are fully loaded
+    });
+
+    test('collects and validates JS Bundles loaded on page', async ({
+      page,
+      pageObjects,
+      perfTracker,
+    }) => {
+      perfTracker.captureBundleResponses(cdp); // Start tracking
+
+      // Navigate to Discover app
+      await pageObjects.collapsibleNav.clickItem('Discover');
+      const currentUrl = page.url();
+      expect(currentUrl).toContain('app/discover#/');
+
+      // Ensure all JS bundles are loaded
+      await perfTracker.waitForJsLoad(cdp);
+
+      // Collect and validate stats
+      const stats = perfTracker.collectJsBundleStats(currentUrl);
+      expect(
+        stats.totalSize,
+        `Total bundles size loaded on page should not exceed 3.0 MB`
+      ).toBeLessThan(3 * 1024 * 1024);
+      expect(stats.bundleCount, {
+        message: `Total bundle chunks count loaded on page should not exceed 100`,
+      }).toBeLessThan(100);
+      expect(
+        stats.plugins.map((p) => p.name),
+        { message: 'Unexpected plugins were loaded on page' }
+      ).toStrictEqual([
+        'aiops',
+        'discover',
+        'eventAnnotation',
+        'expressionXY',
+        'kbn-ui-shared-deps-npm',
+        'lens',
+        'maps',
+        'unifiedHistogram',
+        'unifiedSearch',
+      ]);
+      // Validate individual plugin bundle sizes
+      expect(stats.plugins.find((p) => p.name === 'discover')?.totalSize, {
+        message: `Total 'discover' bundles size should not exceed 625 KB`,
+      }).toBeLessThan(625 * 1024);
+    });
+```
+
+### Uage: collecting CDP Performance metrics
+
+```ts
+test.describe(
+  'Discover App - Performance Metrics & Bundle Analysis',
+  { tag: [...tags.DEPLOYMENT_AGNOSTIC, ...tags.PERFORMANCE] },
+  () => {
+    let cdp: CDPSession;
+
+    test.beforeEach(async ({ browserAuth, page, context, perfTracker }) => {
+      await browserAuth.loginAsAdmin();
+      cdp = await context.newCDPSession(page);
+      // load the starting page, e.g. '/app/home' and wait for loading to finish
+    });
+
+    test('measures Performance Metrics before and after Discover load', async ({
+      page,
+      pageObjects,
+      perfTracker,
+    }) => {
+      const beforeMetrics = await perfTracker.capturePagePerformanceMetrics(cdp);
+
+      // Navigate to Discover app
+      await pageObjects.collapsibleNav.clickItem('Discover');
+      await page.waitForLoadingIndicatorHidden();
+      const currentUrl = page.url();
+      expect(currentUrl).toContain('app/discover#/');
+
+      await pageObjects.discover.waitForHistogramRendered();
+
+      const afterMetrics = await perfTracker.capturePagePerformanceMetrics(cdp);
+      const perfStats = perfTracker.collectPagePerformanceStats(
+        currentUrl,
+        beforeMetrics,
+        afterMetrics
+      );
+
+      expect(perfStats.cpuTime.diff).toBeLessThan(1.5); // CPU time (seconds) usage during page navigation
+      expect(perfStats.scriptTime.diff).toBeLessThan(0.4); // Additional time (seconds) spent executing JS scripts
+      expect(perfStats.layoutTime.diff).toBeLessThan(0.06); // Total layout computation time (seconds)
+    });
+```

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/test/performance/index.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { coreWorkerFixtures } from '../../worker';
+import { PerformanceTracker } from './performance_tracker';
+
+export const perfTrackerFixture = coreWorkerFixtures.extend<{ perfTracker: PerformanceTracker }>({
+  perfTracker: [
+    async ({ log }, use, testInfo) => {
+      log.serviceLoaded('perfTracker');
+
+      const tracker = new PerformanceTracker(testInfo);
+      await use(tracker);
+    },
+    { scope: 'test' },
+  ],
+});
+
+export type PerfTrackerFixture = ReturnType<typeof perfTrackerFixture>;