devops: re-factor list-dependencies script to output per-browser results

This refactors script to output per-browser package dependencies that
could be easily copied over to the Github Action.

References microsoft#2745

Author: aslushnikov

src/server/validateDependencies.ts

@@ -204,7 +204,6 @@ const LIBRARY_TO_PACKAGE_NAME_UBUNTU_18_04: { [s: string]: string} = {
   'libpng16.so.16': 'libpng16-16',
   'libsecret-1.so.0': 'libsecret-1-0',
   'libsmime3.so': 'libnss3',
-  'libssl3.so': 'libnss3',
   'libvpx.so.5': 'libvpx5',
   'libwayland-client.so.0': 'libwayland-client0',
   'libwayland-egl.so.1': 'libwayland-egl1',
@@ -221,85 +220,85 @@ const LIBRARY_TO_PACKAGE_NAME_UBUNTU_18_04: { [s: string]: string} = {
 };
 
 const LIBRARY_TO_PACKAGE_NAME_UBUNTU_20_04: { [s: string]: string} = {
-  'libglib-2.0.so.0': 'libglib2.0-0',
-  'libX11.so.6': 'libx11-6',
-  'libxcb.so.1': 'libxcb1',
-  'libGL.so.1': 'libgl1',
   'libEGL.so.1': 'libegl1',
-  'libnotify.so.4': 'libnotify4',
-  'libgdk_pixbuf-2.0.so.0': 'libgdk-pixbuf2.0-0',
-  'libgio-2.0.so.0': 'libglib2.0-0',
-  'libgobject-2.0.so.0': 'libglib2.0-0',
-  'libvpx.so.6': 'libvpx6',
-  'libopus.so.0': 'libopus0',
-  'libxml2.so.2': 'libxml2',
-  'libicui18n.so.66': 'libicu66',
-  'libicuuc.so.66': 'libicu66',
-  'libxslt.so.1': 'libxslt1.1',
-  'libwoff2dec.so.1.0.2': 'libwoff1',
+  'libGL.so.1': 'libgl1',
+  'libX11-xcb.so.1': 'libx11-xcb1',
+  'libX11.so.6': 'libx11-6',
+  'libXcomposite.so.1': 'libxcomposite1',
+  'libXcursor.so.1': 'libxcursor1',
+  'libXdamage.so.1': 'libxdamage1',
+  'libXext.so.6': 'libxext6',
+  'libXfixes.so.3': 'libxfixes3',
+  'libXi.so.6': 'libxi6',
+  'libXrandr.so.2': 'libxrandr2',
+  'libXrender.so.1': 'libxrender1',
+  'libXt.so.6': 'libxt6',
+  'libXtst.so.6': 'libxtst6',
+  'libasound.so.2': 'libasound2',
+  'libatk-1.0.so.0': 'libatk1.0-0',
+  'libatk-bridge-2.0.so.0': 'libatk-bridge2.0-0',
+  'libatspi.so.0': 'libatspi2.0-0',
+  'libcairo-gobject.so.2': 'libcairo-gobject2',
   'libcairo.so.2': 'libcairo2',
+  'libcups.so.2': 'libcups2',
+  'libdbus-1.so.3': 'libdbus-1-3',
+  'libdbus-glib-1.so.2': 'libdbus-glib-1-2',
+  'libdrm.so.2': 'libdrm2',
+  'libenchant.so.1': 'libenchant1c2a',
+  'libepoxy.so.0': 'libepoxy0',
   'libfontconfig.so.1': 'libfontconfig1',
   'libfreetype.so.6': 'libfreetype6',
-  'libharfbuzz.so.0': 'libharfbuzz0b',
-  'libharfbuzz-icu.so.0': 'libharfbuzz-icu0',
+  'libgbm.so.1': 'libgbm1',
+  'libgdk-3.so.0': 'libgtk-3-0',
+  'libgdk-x11-2.0.so.0': 'libgtk2.0-0',
+  'libgdk_pixbuf-2.0.so.0': 'libgdk-pixbuf2.0-0',
+  'libgio-2.0.so.0': 'libglib2.0-0',
+  'libglib-2.0.so.0': 'libglib2.0-0',
+  'libgmodule-2.0.so.0': 'libglib2.0-0',
+  'libgobject-2.0.so.0': 'libglib2.0-0',
   'libgstapp-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
+  'libgstaudio-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
   'libgstbase-1.0.so.0': 'libgstreamer1.0-0',
-  'libgstreamer-1.0.so.0': 'libgstreamer1.0-0',
+  'libgstcodecparsers-1.0.so.0': 'libgstreamer-plugins-bad1.0-0',
+  'libgstfft-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
+  'libgstgl-1.0.so.0': 'libgstreamer-gl1.0-0',
   'libgstpbutils-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
-  'libgstaudio-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
+  'libgstreamer-1.0.so.0': 'libgstreamer1.0-0',
   'libgsttag-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
   'libgstvideo-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
-  'libgstgl-1.0.so.0': 'libgstreamer-gl1.0-0',
-  'libgstcodecparsers-1.0.so.0': 'libgstreamer-plugins-bad1.0-0',
-  'libgstfft-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
+  'libgthread-2.0.so.0': 'libglib2.0-0',
+  'libgtk-3.so.0': 'libgtk-3-0',
+  'libgtk-x11-2.0.so.0': 'libgtk2.0-0',
+  'libharfbuzz-icu.so.0': 'libharfbuzz-icu0',
+  'libharfbuzz.so.0': 'libharfbuzz0b',
+  'libhyphen.so.0': 'libhyphen0',
+  'libicui18n.so.66': 'libicu66',
+  'libicuuc.so.66': 'libicu66',
   'libjpeg.so.8': 'libjpeg-turbo8',
-  'libpng16.so.16': 'libpng16-16',
+  'libnotify.so.4': 'libnotify4',
+  'libnspr4.so': 'libnspr4',
+  'libnss3.so': 'libnss3',
+  'libnssutil3.so': 'libnss3',
   'libopenjp2.so.7': 'libopenjp2-7',
-  'libwebpdemux.so.2': 'libwebpdemux2',
-  'libwebp.so.6': 'libwebp6',
-  'libsoup-2.4.so.1': 'libsoup2.4-1',
-  'libenchant.so.1': 'libenchant1c2a',
-  'libgmodule-2.0.so.0': 'libglib2.0-0',
-  'libsecret-1.so.0': 'libsecret-1-0',
-  'libhyphen.so.0': 'libhyphen0',
-  'libXcomposite.so.1': 'libxcomposite1',
-  'libXdamage.so.1': 'libxdamage1',
-  'libwayland-server.so.0': 'libwayland-server0',
-  'libwayland-egl.so.1': 'libwayland-egl1',
-  'libwayland-client.so.0': 'libwayland-client0',
-  'libgtk-3.so.0': 'libgtk-3-0',
-  'libgdk-3.so.0': 'libgtk-3-0',
+  'libopus.so.0': 'libopus0',
   'libpango-1.0.so.0': 'libpango-1.0-0',
-  'libatk-1.0.so.0': 'libatk1.0-0',
-  'libxkbcommon.so.0': 'libxkbcommon0',
-  'libepoxy.so.0': 'libepoxy0',
-  'libatk-bridge-2.0.so.0': 'libatk-bridge2.0-0',
-  'libX11-xcb.so.1': 'libx11-xcb1',
-  'libXcursor.so.1': 'libxcursor1',
-  'libXext.so.6': 'libxext6',
-  'libXfixes.so.3': 'libxfixes3',
-  'libXi.so.6': 'libxi6',
-  'libXrender.so.1': 'libxrender1',
-  'libdbus-glib-1.so.2': 'libdbus-glib-1-2',
-  'libdbus-1.so.3': 'libdbus-1-3',
   'libpangocairo-1.0.so.0': 'libpangocairo-1.0-0',
-  'libcairo-gobject.so.2': 'libcairo-gobject2',
-  'libxcb-shm.so.0': 'libxcb-shm0',
   'libpangoft2-1.0.so.0': 'libpangoft2-1.0-0',
-  'libXt.so.6': 'libxt6',
-  'libgthread-2.0.so.0': 'libglib2.0-0',
-  'libgtk-x11-2.0.so.0': 'libgtk2.0-0',
-  'libgdk-x11-2.0.so.0': 'libgtk2.0-0',
-  'libnss3.so': 'libnss3',
-  'libnssutil3.so': 'libnss3',
+  'libpng16.so.16': 'libpng16-16',
+  'libsecret-1.so.0': 'libsecret-1-0',
   'libsmime3.so': 'libnss3',
-  'libnspr4.so': 'libnspr4',
+  'libsoup-2.4.so.1': 'libsoup2.4-1',
+  'libvpx.so.6': 'libvpx6',
+  'libwayland-client.so.0': 'libwayland-client0',
+  'libwayland-egl.so.1': 'libwayland-egl1',
+  'libwayland-server.so.0': 'libwayland-server0',
+  'libwebp.so.6': 'libwebp6',
+  'libwebpdemux.so.2': 'libwebpdemux2',
+  'libwoff2dec.so.1.0.2': 'libwoff1',
   'libxcb-dri3.so.0': 'libxcb-dri3-0',
-  'libXtst.so.6': 'libxtst6',
-  'libcups.so.2': 'libcups2',
-  'libdrm.so.2': 'libdrm2',
-  'libXrandr.so.2': 'libxrandr2',
-  'libgbm.so.1': 'libgbm1',
-  'libasound.so.2': 'libasound2',
-  'libatspi.so.0': 'libatspi2.0-0',
+  'libxcb-shm.so.0': 'libxcb-shm0',
+  'libxcb.so.1': 'libxcb1',
+  'libxkbcommon.so.0': 'libxkbcommon0',
+  'libxml2.so.2': 'libxml2',
+  'libxslt.so.1': 'libxslt1.1',
 };

utils/linux-browser-dependencies/inside_docker/list_dependencies.js

@@ -6,47 +6,157 @@ const path = require('path');
 const {spawn} = require('child_process');
 const browserPaths = require('playwright/lib/install/browserPaths.js');
 
+const readdirAsync = util.promisify(fs.readdir.bind(fs));
+const readFileAsync = util.promisify(fs.readFile.bind(fs));
+
 (async () => {
+  console.log('Working on:', await getDistributionName());
+  console.log('Started at:', currentTime());
   const allBrowsersPath = browserPaths.browsersPath();
-  const {stdout} = await runCommand('find', [allBrowsersPath, '-executable', '-type', 'f']);
-  // lddPaths - files we want to run LDD against.
-  const lddPaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh'));
-  // List of all shared libraries missing.
-  const missingDeps = new Set();
-  // Multimap: reverse-mapping from shared library to requiring file.
-  const depsToLddPaths = new Map();
-  await Promise.all(lddPaths.map(async lddPath => {
-    const deps = await missingFileDependencies(lddPath);
-    for (const dep of deps) {
-      missingDeps.add(dep);
-      let depsToLdd = depsToLddPaths.get(dep);
-      if (!depsToLdd) {
-        depsToLdd = new Set();
-        depsToLddPaths.set(dep, depsToLdd);
-      }
-      depsToLdd.add(lddPath);
-    }
+  const browserDescriptors = (await readdirAsync(allBrowsersPath)).filter(dir => !dir.startsWith('.')).map(dir => ({
+    name: dir,
+    path: path.join(allBrowsersPath, dir),
+    // All files that we will try to inspect for missing dependencies.
+    filePaths: [],
+    // All libraries that are missing for the browser.
+    missingLibraries: new Set(),
+    // All packages required for the browser.
+    requiredPackages: new Set(),
+    // Libraries that we didn't find a package.
+    unresolvedLibraries: new Set(),
   }));
-  console.log(`==== MISSING DEPENDENCIES: ${missingDeps.size} ====`);
-  console.log([...missingDeps].sort().join('\n'));
 
-  console.log('{');
-  for (const dep of missingDeps) {
-    const packages = await findPackages(dep);
-    if (packages.length === 0) {
-      console.log(`  // UNRESOLVED: ${dep} `);
-      const depsToLdd = depsToLddPaths.get(dep);
-      for (const filePath of depsToLdd)
-        console.log(`  // - required by ${filePath}`);
+  // Collect all missing libraries for all browsers.
+  const allMissingLibraries = new Set();
+  for (const descriptor of browserDescriptors) {
+    const {stdout} = await runCommand('find', [descriptor.path, '-executable', '-type', 'f']);
+    descriptor.filePaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh'));
+    await Promise.all(descriptor.filePaths.map(async filePath => {
+      const missingLibraries = await missingFileDependencies(filePath);
+      for (const library of missingLibraries) {
+        descriptor.missingLibraries.add(library);
+        allMissingLibraries.add(library);
+      }
+    }));
+  }
+
+  const libraryToPackage = new Map();
+  const ambiguityLibraries = new Map();
+
+  // Map missing libraries to packages that could be installed to fulfill the dependency.
+  console.log(`Finding packages for ${allMissingLibraries.size} missing libraries...`);
+
+  for (let i = 0, array = [...allMissingLibraries].sort(); i < allMissingLibraries.size; ++i) {
+    const library = array[i];
+    const packages = await findPackages(library);
+
+    const progress = `${i + 1}/${allMissingLibraries.size}`;
+    console.log(`${progress.padStart(7)}: ${library} => ${JSON.stringify(packages)}`);
+
+    if (!packages.length) {
+      const browsersWithMissingLibrary = browserDescriptors.filter(d => d.missingLibraries.has(library)).map(d => d.name).join(', ');
+      const PADDING = ''.padStart(7) + '  ';
+      console.log(PADDING + `ERROR: failed to resolve '${library}' required by ${browsersWithMissingLibrary}`);
     } else if (packages.length === 1) {
-      console.log(`  "${dep}": "${packages[0]}",`);
+      libraryToPackage.set(library, packages[0]);
     } else {
-      console.log(`  "${dep}": ${JSON.stringify(packages)},`);
+      ambiguityLibraries.set(library, packages);
     }
   }
+
+  console.log('');
+  console.log(`Picking packages for ${ambiguityLibraries.size} libraries that have multiple package candidates`);
+  // Pick packages to install to fulfill missing libraries.
+  //
+  // This is a 2-step process:
+  // 1. Pick easy libraries by filtering out debug, test and dev packages.
+  // 2. After that, pick packages that we already picked before.
+
+  // Step 1: pick libraries that are easy to pick.
+  const totalAmbiguityLibraries = ambiguityLibraries.size;
+  for (const [library, packages] of ambiguityLibraries) {
+    const package = pickPackage(library, packages);
+    if (!package)
+      continue;
+    libraryToPackage.set(library, package);
+    ambiguityLibraries.delete(library);
+    const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
+    console.log(`${progress.padStart(7)}: ${library} => ${package}`);
+    console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
+  }
+  // 2nd pass - prefer packages that we already picked.
+  const allUsedPackages = new Set(libraryToPackage.values());
+  for (const [library, packages] of ambiguityLibraries) {
+    const package = packages.find(package => allUsedPackages.has(package));
+    if (!package)
+      continue;
+    libraryToPackage.set(library, package);
+    ambiguityLibraries.delete(library);
+    const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
+    console.log(`${progress.padStart(7)}: ${library} => ${package}`);
+    console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
+  }
+
+  // Report all ambiguities that were failed to resolve.
+  for (const [library, packages] of ambiguityLibraries) {
+    ambiguityLibraries.delete(library);
+    const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
+    console.log(`${progress.padStart(7)}: FAILED ${library} => ???`);
+    console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
+  }
+
+  // For each browser build a list of packages to install.
+  for (const descriptor of browserDescriptors) {
+    for (const library of descriptor.missingLibraries) {
+      const package = libraryToPackage.get(library);
+      if (package)
+        descriptor.requiredPackages.add(package);
+      else
+        descriptor.unresolvedLibraries.add(library);
+    }
+  }
+
+  // Formatting results.
+  console.log('');
+  console.log(`----- Library to package name mapping -----`);
+  console.log('{');
+  for (const [library, package] of libraryToPackage)
+    console.log(`  "${library}": "${package}",`);
   console.log('}');
+
+  // Packages and unresolved libraries for every browser
+  for (const descriptor of browserDescriptors) {
+    console.log('');
+    console.log(`======= ${descriptor.name}:  required packages =======`);
+    const requiredPackages = [...descriptor.requiredPackages].sort();
+    console.log(JSON.stringify(requiredPackages, null, 2));
+    console.log('');
+    console.log(`------- ${descriptor.name}:  unresolved libraries -------`);
+    const unresolvedLibraries = [...descriptor.unresolvedLibraries].sort();
+    console.log(JSON.stringify(unresolvedLibraries, null, 2));
+  }
+
+  const status = browserDescriptors.some(d => d.unresolvedLibraries.size) ? 'FAILED' : 'SUCCESS';
+  console.log(`
+  ====================
+        ${status}
+  ====================
+  `);
 })();
 
+function pickPackage(library, packages) {
+  // Step 1: try to filter out debug, test and dev packages.
+  packages = packages.filter(package => !package.endsWith('-dbg') && !package.endsWith('-test') && !package.endsWith('-dev') && !package.endsWith('-mesa'));
+  if (packages.length === 1)
+    return packages[0];
+  // Step 2: use library name to filter packages with the same name.
+  const prefix = library.split(/[-.]/).shift().toLowerCase();
+  packages = packages.filter(package => package.toLowerCase().startsWith(prefix));
+  if (packages.length === 1)
+    return packages[0];
+  return null;
+}
+
 async function findPackages(libraryName) {
   const {stdout} = await runCommand('apt-file', ['search', libraryName]);
   if (!stdout.trim())
@@ -56,7 +166,9 @@ async function findPackages(libraryName) {
 }
 
 async function fileDependencies(filePath) {
-  const {stdout} = await lddAsync(filePath);
+  const {stdout, code} = await lddAsync(filePath);
+  if (code !== 0)
+    return [];
   const deps = stdout.split('\n').map(line => {
     line = line.trim();
     const missing = line.includes('not found');
@@ -100,3 +212,26 @@ function runCommand(command, args, options = {}) {
     });
   });
 }
+
+async function getDistributionName() {
+  const osReleaseText = await readFileAsync('/etc/os-release', 'utf8');
+  const fields = new Map();
+  for (const line of osReleaseText.split('\n')) {
+    const tokens = line.split('=');
+    const name = tokens.shift();
+    let value = tokens.join('=').trim();
+    if (value.startsWith('"') && value.endsWith('"'))
+      value = value.substring(1, value.length - 1);
+    if (!name)
+      continue;
+    fields.set(name.toLowerCase(), value);
+  }
+  return fields.get('pretty_name') || '';
+}
+
+function currentTime() {
+  const date = new Date();
+  const dateTimeFormat = new Intl.DateTimeFormat('en', { year: 'numeric', month: 'short', day: '2-digit' });
+  const [{ value: month },,{ value: day },,{ value: year }] = dateTimeFormat .formatToParts(date );
+  return `${month} ${day}, ${year}`;
+}