[8.7] [Reporting/Screenshotting] upgrade Puppeteer dependency to v20.1.0 (#156766)

package.json

@@ -564,7 +564,7 @@
     "prop-types": "^15.8.1",
     "proxy-from-env": "1.0.0",
     "puid": "1.0.7",
-    "puppeteer": "18.2.1",
+    "puppeteer": "20.1.0",
     "query-string": "^6.13.2",
     "rbush": "^3.0.1",
     "re-resizable": "^6.1.1",

src/dev/build/tasks/install_chromium.js → src/dev/build/tasks/install_chromium.ts

@@ -6,9 +6,11 @@
  * Side Public License, v 1.
  */
 
+import type { Logger } from '@kbn/core/server';
 import { install, paths } from '@kbn/screenshotting-plugin/server/utils';
+import type { Task } from '../lib';
 
-export const InstallChromium = {
+export const InstallChromium: Task = {
   description: 'Installing Chromium',
 
   async run(config, log, build) {
@@ -40,7 +42,7 @@ export const InstallChromium = {
       };
 
       const path = build.resolvePathForPlatform(platform, 'x-pack/plugins/screenshotting/chromium');
-      await install(logger, pkg, path);
+      await install(logger as unknown as Logger, pkg, path);
     }
   },
 };

src/dev/chromium_version.ts → src/dev/chromium_version/index.ts

@@ -7,30 +7,31 @@
  */
 
 import { run } from '@kbn/dev-cli-runner';
-import { ToolingLog } from '@kbn/tooling-log';
 import { REPO_ROOT } from '@kbn/repo-info';
+import { ToolingLog } from '@kbn/tooling-log';
 import chalk from 'chalk';
-import cheerio from 'cheerio';
 import fs from 'fs';
 import fetch from 'node-fetch';
 import path from 'path';
-
-type PuppeteerRelease = string;
-type ChromiumRevision = string;
-type ChromiumCommit = string;
-
-// We forked the Puppeteer node module for Kibana,
-// So we need to translate OUR version to the official Puppeteer Release
-const forkCompatibilityMap: Record<string, PuppeteerRelease> = {
-  '5.4.1-patch.1': '5.4.1',
-};
+import {
+  type ChromeVersion,
+  type ChromiumCommit,
+  type ChromiumDashVersionType,
+  ChromiumDashVersionSchema,
+  forkCompatibilityMap,
+  PuppeteerPackageSchema,
+  type PuppeteerPackageType,
+  type PuppeteerRelease,
+} from './util';
 
 async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
   // open node_modules/puppeteer/package.json
-  const puppeteerPackageJson = JSON.parse(
+  const { version }: PuppeteerPackageType = JSON.parse(
     fs.readFileSync(path.resolve(REPO_ROOT, 'node_modules', 'puppeteer', 'package.json'), 'utf8')
   );
-  const { version } = puppeteerPackageJson;
+
+  PuppeteerPackageSchema.validate({ version });
+
   if (version == null) {
     throw new Error(
       'Could not get the Puppeteer version! Check node_modules/puppteer/package.json'
@@ -42,11 +43,11 @@ async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
   return puppeteerRelease;
 }
 
-async function getChromiumRevision(
+async function getChromeVersion(
   kibanaPuppeteerVersion: PuppeteerRelease,
   log: ToolingLog
-): Promise<ChromiumRevision> {
-  const url = `https://raw.githubusercontent.com/puppeteer/puppeteer/v${kibanaPuppeteerVersion}/packages/puppeteer-core/src/revisions.ts`;
+): Promise<ChromeVersion> {
+  const url = `https://raw.githubusercontent.com/puppeteer/puppeteer/puppeteer-v${kibanaPuppeteerVersion}/packages/puppeteer-core/src/revisions.ts`;
   let body: string;
   try {
     log.info(`Fetching code from Puppeteer source: ${url}`);
@@ -57,58 +58,51 @@ async function getChromiumRevision(
     throw new Error(`Could not fetch ${url}. Check the URL in a browser and try again.`);
   }
 
-  let revision: ChromiumRevision | undefined;
+  let version: ChromeVersion | undefined;
   const lines = body.split('\n');
   let cursor = lines.length;
   while (--cursor >= 0) {
-    // look for the line of code matching `  chromium: '0123456',`
-    const test = lines[cursor].match(/^\s+chromium: '(\S+)',$/);
+    // look for the line of code matching `  chrome: '113.0.5672.63',`
+    const test = lines[cursor].match(/^\s+chrome: '(\S+)',$/);
     if (test != null) {
-      log.debug(`Parsed revision from source text: \`${lines[cursor]}\``);
-      [, revision] = test;
+      log.debug(`Parsed Chrome version from source text: \`${lines[cursor]}\``);
+      [, version] = test;
       break;
     }
   }
 
-  if (revision == null) {
+  if (version == null) {
     throw new Error(
-      `Could not find a Chromium revision listed in Puppeteer source! Check ${url} in a browser`
+      `Could not find a Chrome version listed in Puppeteer source! Check ${url} in a browser`
     );
   }
 
-  log.info(`Found Chromium revision ${revision} from Puppeteer ${kibanaPuppeteerVersion}`);
-  return revision;
+  log.info(`Found Chrome version ${version} from Puppeteer ${kibanaPuppeteerVersion}`);
+  return version;
 }
 
-async function getChromiumCommit(
-  revision: ChromiumRevision,
-  log: ToolingLog
-): Promise<ChromiumCommit> {
-  const url = `https://crrev.com/${revision}`;
+async function getChromiumCommit(version: ChromeVersion, log: ToolingLog): Promise<ChromiumCommit> {
+  const url = `https://chromiumdash.appspot.com/fetch_version?version=${version}`;
   log.info(`Fetching ${url}`);
-  const pageText = await fetch(url);
-  const $ = cheerio.load(await pageText.text());
-
-  // get the commit from the page title
-  let commit: ChromiumCommit | null = null;
-  const matches = $('title')
-    .text()
-    .match(/\S{40}/);
-  if (matches != null) {
-    log.debug(`Parsed commit hash from page title: \`${$('title').text()}\``);
-    [commit] = matches;
-  }
+  const fetchResponse = await fetch(url);
+  const chromeJson: ChromiumDashVersionType = await fetchResponse.json();
+
+  const {
+    chromium_main_branch_position: revision,
+    hashes: { chromium: commit },
+  } = chromeJson;
+
+  ChromiumDashVersionSchema.validate({
+    chromium_main_branch_position: revision,
+    hashes: { chromium: commit },
+  });
 
   if (commit == null) {
     throw new Error(`Could not find a Chromium commit! Check ${url} in a browser.`);
   }
 
-  const baseUrl = 'https://commondatastorage.googleapis.com/chromium-browser-snapshots';
-
-  log.info(`Found Chromium commit ${commit} from revision ${revision}.`);
-  log.info(`Mac x64 download:     ${baseUrl}/Mac/${revision}/chrome-mac.zip`);
-  log.info(`Mac ARM download:     ${baseUrl}/Mac_Arm/${revision}/chrome-mac.zip`);
-  log.info(`Windows x64 download: ${baseUrl}/Win/${revision}/chrome-win.zip`);
+  log.info(`Found Chromium revision ${revision} from version ${version}.`);
+  log.info(`Found Chromium commit   ${commit} from revision ${revision}.`);
   return commit;
 }
 
@@ -127,8 +121,8 @@ run(
         puppeteerVersion = await getPuppeteerRelease(log);
       }
 
-      const chromiumRevision = await getChromiumRevision(puppeteerVersion, log);
-      await getChromiumCommit(chromiumRevision, log);
+      const chromeVersion = await getChromeVersion(puppeteerVersion, log);
+      await getChromiumCommit(chromeVersion, log);
     } catch (err) {
       log.error(err);
     }

src/dev/chromium_version/util.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export type PuppeteerRelease = string;
+export type ChromeVersion = string;
+export type ChromiumCommit = string;
+
+export type PuppeteerPackageType = TypeOf<typeof PuppeteerPackageSchema>;
+export type ChromiumDashVersionType = TypeOf<typeof ChromiumDashVersionSchema>;
+
+export const PuppeteerPackageSchema = schema.object({
+  version: schema.string(),
+});
+
+export const ChromiumDashVersionSchema = schema.object({
+  chromium_main_branch_position: schema.number(),
+  hashes: schema.object({
+    chromium: schema.string({ minLength: 40, maxLength: 40 }),
+  }),
+});
+
+// We forked the Puppeteer node module for Kibana,
+// So we need to translate OUR version to the official Puppeteer Release
+export const forkCompatibilityMap: Record<string, PuppeteerRelease> = {
+  '5.4.1-patch.1': '5.4.1',
+};

src/dev/tsconfig.json

@@ -35,5 +35,6 @@
     "@kbn/find-used-node-modules",
     "@kbn/ts-projects",
     "@kbn/repo-packages",
+    "@kbn/config-schema",
   ]
 }

x-pack/build_chromium/linux/args.gn

@@ -24,4 +24,4 @@ enable_vr = false
 use_kerberos = false
 
 target_os = "linux"
-# target_cpu is added at build timeure a minimal build
+# target_cpu is added at build time

x-pack/plugins/screenshotting/server/browsers/chromium/driver.ts

@@ -12,7 +12,8 @@ import {
 } from '@kbn/screenshot-mode-plugin/server';
 import { truncate } from 'lodash';
 import open from 'opn';
-import puppeteer, { ElementHandle, Page, EvaluateFunc } from 'puppeteer';
+import * as puppeteer from 'puppeteer';
+import { ElementHandle, EvaluateFunc, Page } from 'puppeteer';
 import { Subject } from 'rxjs';
 import { parse as parseUrl } from 'url';
 import { getDisallowedOutgoingUrlError } from '.';

x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/args.test.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import os from 'os';
+import { args } from './args';
+import { getChromiumPackage } from '../../../utils';
+
+// Since chromium v111 headless mode in arm based macs is not working with `--disable-gpu`
+// This is a known issue: headless uses swiftshader by default and swiftshader's support for WebGL is currently disabled on Arm pending the resolution of https://issuetracker.google.com/issues/165000222.
+// As a workaround, we force hardware GL drivers on mac.
+// The best way to do this starting with v112 is by passing --enable-gpu,
+// v111 and older versions should work with --use-angle.
+describe('headless webgl arm mac workaround', () => {
+  const originalPlatform = process.platform;
+  afterEach(() => {
+    Object.defineProperty(process, 'platform', {
+      value: originalPlatform,
+    });
+  });
+
+  const simulateEnv = (platform: string, arch: string) => {
+    Object.defineProperty(process, 'platform', { value: platform });
+    jest.spyOn(os, 'arch').mockReturnValue(arch);
+  };
+
+  test('disables gpu for non arm mac', () => {
+    simulateEnv('darwin', 'x64');
+
+    const flags = args({
+      userDataDir: '/',
+      proxy: { enabled: false },
+    });
+    expect(flags.includes(`--disable-gpu`)).toBe(true);
+  });
+
+  test("doesn't disable gpu when on an arm mac, adds --use-angle", () => {
+    simulateEnv('darwin', 'arm64');
+
+    const flags = args({
+      userDataDir: '/',
+      proxy: { enabled: false },
+    });
+    expect(flags.includes(`--disable-gpu`)).toBe(false);
+    expect(flags.includes(`--use-angle`)).toBe(true);
+
+    // if you're updating this, then you're likely updating chromium
+    // please double-check that the --use-angle flag is still needed for arm macs
+    // instead of --use-angle you may need --enable-gpu
+    expect(getChromiumPackage().binaryChecksum).toBe('4cc4ee072b23e4a65e714ff543eea21b'); // just putting this here so that someone updating the chromium version will see this comment
+  });
+});

x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/args.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import os from 'os';
 import type { ConfigType } from '../../../config';
 
 interface WindowSize {
@@ -48,7 +49,6 @@ export const args = ({
     // Skip first run wizards
     '--no-first-run',
     `--user-data-dir=${userDataDir}`,
-    '--disable-gpu',
     '--headless',
     '--hide-scrollbars',
     // allow screenshot clip region to go outside of the viewport
@@ -73,7 +73,17 @@ export const args = ({
     flags.push('--no-sandbox');
   }
 
-  if (process.platform === 'linux') {
+  // Since chromium v111 headless mode in arm based macs is not working with `--disable-gpu`
+  // This is a known issue: headless uses swiftshader by default and swiftshader's support for WebGL is currently disabled on Arm pending the resolution of https://issuetracker.google.com/issues/165000222.
+  // As a workaround, we force hardware GL drivers on mac: v111 and older versions should work with --use-angle.
+  // The best way to do this when the issue is resolved will be to pass --enable-gpu,
+  if (os.arch() === 'arm64' && process.platform === 'darwin') {
+    flags.push('--use-angle');
+  } else {
+    flags.push('--disable-gpu');
+  }
+
+  if (os.arch() === 'linux') {
     flags.push('--disable-setuid-sandbox');
   }
 

x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.test.ts

@@ -7,7 +7,7 @@
 
 import type { Logger } from '@kbn/core/server';
 import type { ScreenshotModePluginSetup } from '@kbn/screenshot-mode-plugin/server';
-import puppeteer from 'puppeteer';
+import * as puppeteer from 'puppeteer';
 import * as Rx from 'rxjs';
 import { mergeMap, take } from 'rxjs/operators';
 import { DEFAULT_VIEWPORT, HeadlessChromiumDriverFactory } from '.';

x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.ts

@@ -166,6 +166,7 @@ export class HeadlessChromiumDriverFactory {
             env: {
               TZ: browserTimezone,
             },
+            headless: 'new',
           });
         } catch (err) {
           observer.error(