[Reporting/Screenshotting] upgrade Puppeteer dependency to v20.1.0

package.json

@@ -880,7 +880,7 @@
     "prop-types": "^15.8.1",
     "proxy-from-env": "1.0.0",
     "puid": "1.0.7",
-    "puppeteer": "19.7.2",
+    "puppeteer": "20.1.0",
     "query-string": "^6.13.2",
     "rbush": "^3.0.1",
     "re-resizable": "^6.1.1",

src/dev/build/tasks/install_chromium.js → src/dev/build/tasks/install_chromium.ts

@@ -6,9 +6,11 @@
  * Side Public License, v 1.
  */
 
+import type { Logger } from '@kbn/core/server';
 import { install, paths } from '@kbn/screenshotting-plugin/server/utils';
+import type { Task } from '../lib';
 
-export const InstallChromium = {
+export const InstallChromium: Task = {
   description: 'Installing Chromium',
 
   async run(config, log, build) {
@@ -43,7 +45,7 @@ export const InstallChromium = {
         platform,
         'node_modules/@kbn/screenshotting-plugin/chromium'
       );
-      await install(logger, pkg, path);
+      await install(logger as unknown as Logger, pkg, path);
     }
   },
 };

src/dev/chromium_version.ts → src/dev/chromium_version/index.ts

@@ -7,30 +7,31 @@
  */
 
 import { run } from '@kbn/dev-cli-runner';
-import { ToolingLog } from '@kbn/tooling-log';
 import { REPO_ROOT } from '@kbn/repo-info';
+import { ToolingLog } from '@kbn/tooling-log';
 import chalk from 'chalk';
-import cheerio from 'cheerio';
 import fs from 'fs';
 import fetch from 'node-fetch';
 import path from 'path';
-
-type PuppeteerRelease = string;
-type ChromiumRevision = string;
-type ChromiumCommit = string;
-
-// We forked the Puppeteer node module for Kibana,
-// So we need to translate OUR version to the official Puppeteer Release
-const forkCompatibilityMap: Record<string, PuppeteerRelease> = {
-  '5.4.1-patch.1': '5.4.1',
-};
+import {
+  type ChromeVersion,
+  type ChromiumCommit,
+  type ChromiumDashVersionType,
+  ChromiumDashVersionSchema,
+  forkCompatibilityMap,
+  PuppeteerPackageSchema,
+  type PuppeteerPackageType,
+  type PuppeteerRelease,
+} from './util';
 
 async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
   // open node_modules/puppeteer/package.json
-  const puppeteerPackageJson = JSON.parse(
+  const { version }: PuppeteerPackageType = JSON.parse(
     fs.readFileSync(path.resolve(REPO_ROOT, 'node_modules', 'puppeteer', 'package.json'), 'utf8')
   );
-  const { version } = puppeteerPackageJson;
+
+  PuppeteerPackageSchema.validate({ version });
+
   if (version == null) {
     throw new Error(
       'Could not get the Puppeteer version! Check node_modules/puppteer/package.json'
@@ -42,10 +43,10 @@ async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
   return puppeteerRelease;
 }
 
-async function getChromiumRevision(
+async function getChromeVersion(
   kibanaPuppeteerVersion: PuppeteerRelease,
   log: ToolingLog
-): Promise<ChromiumRevision> {
+): Promise<ChromeVersion> {
   const url = `https://raw.githubusercontent.com/puppeteer/puppeteer/puppeteer-v${kibanaPuppeteerVersion}/packages/puppeteer-core/src/revisions.ts`;
   let body: string;
   try {
@@ -57,58 +58,51 @@ async function getChromiumRevision(
     throw new Error(`Could not fetch ${url}. Check the URL in a browser and try again.`);
   }
 
-  let revision: ChromiumRevision | undefined;
+  let version: ChromeVersion | undefined;
   const lines = body.split('\n');
   let cursor = lines.length;
   while (--cursor >= 0) {
-    // look for the line of code matching `  chromium: '0123456',`
-    const test = lines[cursor].match(/^\s+chromium: '(\S+)',$/);
+    // look for the line of code matching `  chrome: '113.0.5672.63',`
+    const test = lines[cursor].match(/^\s+chrome: '(\S+)',$/);
     if (test != null) {
-      log.debug(`Parsed revision from source text: \`${lines[cursor]}\``);
-      [, revision] = test;
+      log.debug(`Parsed Chrome version from source text: \`${lines[cursor]}\``);
+      [, version] = test;
       break;
     }
   }
 
-  if (revision == null) {
+  if (version == null) {
     throw new Error(
-      `Could not find a Chromium revision listed in Puppeteer source! Check ${url} in a browser`
+      `Could not find a Chrome version listed in Puppeteer source! Check ${url} in a browser`
     );
   }
 
-  log.info(`Found Chromium revision ${revision} from Puppeteer ${kibanaPuppeteerVersion}`);
-  return revision;
+  log.info(`Found Chrome version ${version} from Puppeteer ${kibanaPuppeteerVersion}`);
+  return version;
 }
 
-async function getChromiumCommit(
-  revision: ChromiumRevision,
-  log: ToolingLog
-): Promise<ChromiumCommit> {
-  const url = `https://crrev.com/${revision}`;
+async function getChromiumCommit(version: ChromeVersion, log: ToolingLog): Promise<ChromiumCommit> {
+  const url = `https://chromiumdash.appspot.com/fetch_version?version=${version}`;
   log.info(`Fetching ${url}`);
-  const pageText = await fetch(url);
-  const $ = cheerio.load(await pageText.text());
-
-  // get the commit from the page title
-  let commit: ChromiumCommit | null = null;
-  const matches = $('title')
-    .text()
-    .match(/\S{40}/);
-  if (matches != null) {
-    log.debug(`Parsed commit hash from page title: \`${$('title').text()}\``);
-    [commit] = matches;
-  }
+  const fetchResponse = await fetch(url);
+  const chromeJson: ChromiumDashVersionType = await fetchResponse.json();
+
+  const {
+    chromium_main_branch_position: revision,
+    hashes: { chromium: commit },
+  } = chromeJson;
+
+  ChromiumDashVersionSchema.validate({
+    chromium_main_branch_position: revision,
+    hashes: { chromium: commit },
+  });
 
   if (commit == null) {
     throw new Error(`Could not find a Chromium commit! Check ${url} in a browser.`);
   }
 
-  const baseUrl = 'https://commondatastorage.googleapis.com/chromium-browser-snapshots';
-
-  log.info(`Found Chromium commit ${commit} from revision ${revision}.`);
-  log.info(`Mac x64 download:     ${baseUrl}/Mac/${revision}/chrome-mac.zip`);
-  log.info(`Mac ARM download:     ${baseUrl}/Mac_Arm/${revision}/chrome-mac.zip`);
-  log.info(`Windows x64 download: ${baseUrl}/Win/${revision}/chrome-win.zip`);
+  log.info(`Found Chromium revision ${revision} from version ${version}.`);
+  log.info(`Found Chromium commit   ${commit} from revision ${revision}.`);
   return commit;
 }
 
@@ -127,8 +121,8 @@ run(
         puppeteerVersion = await getPuppeteerRelease(log);
       }
 
-      const chromiumRevision = await getChromiumRevision(puppeteerVersion, log);
-      await getChromiumCommit(chromiumRevision, log);
+      const chromeVersion = await getChromeVersion(puppeteerVersion, log);
+      await getChromiumCommit(chromeVersion, log);
     } catch (err) {
       log.error(err);
     }

src/dev/chromium_version/util.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export type PuppeteerRelease = string;
+export type ChromeVersion = string;
+export type ChromiumCommit = string;
+
+export type PuppeteerPackageType = TypeOf<typeof PuppeteerPackageSchema>;
+export type ChromiumDashVersionType = TypeOf<typeof ChromiumDashVersionSchema>;
+
+export const PuppeteerPackageSchema = schema.object({
+  version: schema.string(),
+});
+
+export const ChromiumDashVersionSchema = schema.object({
+  chromium_main_branch_position: schema.number(),
+  hashes: schema.object({
+    chromium: schema.string({ minLength: 40, maxLength: 40 }),
+  }),
+});
+
+// We forked the Puppeteer node module for Kibana,
+// So we need to translate OUR version to the official Puppeteer Release
+export const forkCompatibilityMap: Record<string, PuppeteerRelease> = {
+  '5.4.1-patch.1': '5.4.1',
+};

src/dev/tsconfig.json

@@ -39,5 +39,6 @@
     "@kbn/get-repo-files",
     "@kbn/import-locator",
     "@kbn/journeys",
+    "@kbn/config-schema",
   ]
 }

x-pack/build_chromium/linux/args.gn

@@ -24,4 +24,4 @@ enable_vr = false
 use_kerberos = false
 
 target_os = "linux"
-# target_cpu is added at build timeure a minimal build
+# target_cpu is added at build time

x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/args.test.ts

@@ -49,7 +49,7 @@ describe('headless webgl arm mac workaround', () => {
 
     // if you're updating this, then you're likely updating chromium
     // please double-check that the --use-angle flag is still needed for arm macs
-    // if you're updating to v112, then likely instead of --use-angle you should use --enable-gpu
-    expect(getChromiumPackage().binaryChecksum).toBe('0a12a34a0d8bc9c616d3cc339abb167e'); // just putting this here so that someone updating the chromium version will see this comment
+    // instead of --use-angle you may need --enable-gpu
+    expect(getChromiumPackage().binaryChecksum).toBe('4cc4ee072b23e4a65e714ff543eea21b'); // just putting this here so that someone updating the chromium version will see this comment
   });
 });

x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/args.ts

@@ -75,9 +75,8 @@ export const args = ({
 
   // Since chromium v111 headless mode in arm based macs is not working with `--disable-gpu`
   // This is a known issue: headless uses swiftshader by default and swiftshader's support for WebGL is currently disabled on Arm pending the resolution of https://issuetracker.google.com/issues/165000222.
-  // As a workaround, we force hardware GL drivers on mac.
-  // The best way to do this starting with v112 is by passing --enable-gpu,
-  // v111 and older versions should work with --use-angle.
+  // As a workaround, we force hardware GL drivers on mac: v111 and older versions should work with --use-angle.
+  // The best way to do this when the issue is resolved will be to pass --enable-gpu,
   if (os.arch() === 'arm64' && process.platform === 'darwin') {
     flags.push('--use-angle');
   } else {

x-pack/plugins/screenshotting/server/browsers/chromium/integration_tests/downloads.test.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
+import assert from 'assert';
+import axios from 'axios';
+import path from 'path';
+import { PackageInfo } from '..';
+import { paths as chromiumArchivePaths } from '../../../utils';
+import { download } from '../../download';
+import { install } from '../../install';
+
+/* eslint-disable no-console */
+
+const mockLogger = loggingSystemMock.create().get();
+mockLogger.warn = jest.fn((message: string | Error) => {
+  console.warn(message);
+});
+mockLogger.debug = jest.fn((message: string | Error) => {
+  console.log(message);
+});
+mockLogger.error = jest.fn((message: string | Error) => {
+  console.error(message);
+});
+
+/**
+ * NOTE: these test cases download actual browsers. Running the suite could take
+ * a long time with a clean cache.
+ */
+
+// test case tuples
+const packageInfos = chromiumArchivePaths.packages.map(({ platform, architecture }) => [
+  architecture,
+  platform,
+]);
+
+describe.each(packageInfos)('Chromium archive: %s/%s', (architecture, platform) => {
+  // For testing, suffix the unzip folder by cpu + platform so the extracted folders do not overwrite each other in the cache
+  const chromiumPath = path.resolve(__dirname, '../../../../chromium', architecture, platform);
+
+  const originalAxios = axios.defaults.adapter;
+  beforeAll(async () => {
+    axios.defaults.adapter = require('axios/lib/adapters/http'); // allow Axios to send actual requests
+  });
+
+  afterAll(() => {
+    axios.defaults.adapter = originalAxios;
+  });
+
+  // Allow package definition to be altered to check error handling
+  const originalPkg = chromiumArchivePaths.packages.find(
+    (packageInfo) => packageInfo.platform === platform && packageInfo.architecture === architecture
+  );
+  assert(originalPkg);
+
+  let pkg: PackageInfo = originalPkg;
+  beforeEach(() => {
+    pkg = { ...originalPkg };
+  });
+
+  it('references the correct checksums and binary path', async () => {
+    const downloadedChecksum = await download(chromiumArchivePaths, pkg, mockLogger);
+    expect(downloadedChecksum).toBe(pkg.archiveChecksum);
+
+    const binaryPath = await install(chromiumArchivePaths, mockLogger, pkg, chromiumPath);
+    expect(binaryPath).toBe(path.join(chromiumPath, pkg.binaryRelativePath));
+  });
+});