promote: add check_r2_assets tool (#3977)

Add tool and tests for checking if assets on R2 look ready for
promotion. Relies on `rclone` being available -- there's a test for
the case where it is not.

The new tool is based on existing `check_assets.js` but:
- Written in ESM.
- Uses built-in Set objects instead of emulating sets with Arrays.
- Always assumes an asset file exists. `check_assets.js` has some
  fallback logic, but it's not straightforward and nowadays we can
  consider a missing asset file to be a process failure.
- Reflects that `.done` files are not used for R2 (basically they
  don't exist so no handling logic). Also note that for R2,
  promotion is a copy and not a move so after a partial promotion
  subsequent checks will result in overwrite warnings for files
  previously promoted.
- `SHASUMS256.txt` can end up in the staging directory. Ignore for
  the staging directory as `check_assets.js` does for dist. (The
  shasums are generated by later parts of the release process.)

Fixes: #3958

.github/workflows/check_assets-tool.yml

@@ -6,11 +6,15 @@ on:
       - '.github/workflows/check_assets-tool.yml'
       - 'ansible/www-standalone/tools/promote/expected_assets/*'
       - 'ansible/www-standalone/tools/promote/check_assets*'
+      - 'ansible/www-standalone/tools/promote/check_r2_assets*'
+      - 'ansible/www-standalone/tools/promote/test/**'
   push:
     paths:
       - '.github/workflows/check_assets-tool.yml'
       - 'ansible/www-standalone/tools/promote/expected_assets/*'
       - 'ansible/www-standalone/tools/promote/check_assets*'
+      - 'ansible/www-standalone/tools/promote/check_r2_assets*'
+      - 'ansible/www-standalone/tools/promote/test/**'
   schedule:
     - cron: 0 0 * * *
   workflow_dispatch:
@@ -35,5 +39,5 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
       - name: Run tests
-        run: node --test
+        run: node --test --experimental-test-module-mocks
         working-directory: ansible/www-standalone/tools/promote/  

ansible/www-standalone/tools/promote/check_r2_assets.mjs

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+
+import { exec } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+
+const versionRe = /^v\d+\.\d+\.\d+/
+// These are normally generated as part of the release process after the asset
+// check, but may be present if a release has already been partially promoted.
+const additionalAssets = new Set([
+  'SHASUMS256.txt',
+  'SHASUMS256.txt.asc',
+  'SHASUMS256.txt.sig'
+]);
+
+if (process.argv[1] === import.meta.filename) {
+  checkArgs(process.argv).then(run(process.argv[2], process.argv[3])).catch(console.error);
+}
+
+async function checkArgs (argv) {
+  let bad = false;
+  if (!argv || argv.length < 4) {
+    bad = true;
+  } else {
+    if (!versionRe.test(basename(argv[2]))) {
+      bad = true;
+      console.error(`Bad staging directory name: ${argv[2]}`);
+    }
+    if (!versionRe.test(basename(argv[3]))) {
+      bad = true;
+      console.error(`Bad dist directory name: ${argv[3]}`);
+    }
+  }
+  if (bad) {
+    console.error(`Usage: ${basename(import.meta.filename)} <path to staging directory> <path to dist directory>`);
+    process.exit(1);
+  }
+}
+
+async function loadExpectedAssets (version, line) {
+  try {
+    const templateFile = join(import.meta.dirname, 'expected_assets', line);
+    let files = await readFile(templateFile, 'utf8');
+    return files.replace(/{VERSION}/g, version).split(/\n/g).filter(Boolean);
+  } catch (e) { }
+  return null;
+}
+
+async function lsRemoteDepth2 (dir) {
+  return new Promise((resolve, reject) => {
+    const command = `rclone lsjson ${dir} --no-modtime --no-mimetype -R --max-depth 2`;
+    exec(command, {}, (err, stdout, stderr) => {
+      if (err) {
+        return reject(err);
+      }
+      if (stderr) {
+        console.log('STDERR:', stderr);
+      }
+      const assets = JSON.parse(stdout).map(({ Path, IsDir }) => {
+        if (IsDir) {
+          return `${Path}/`;
+        }
+        return Path;
+      })
+      resolve(assets);
+    });
+  });
+}
+
+async function run (stagingDir, distDir) {
+  const version = basename(stagingDir);
+  const line = versionToLine(version);
+  const stagingAssets = new Set(await lsRemoteDepth2(stagingDir)).difference(additionalAssets);
+  const distAssets = new Set((await lsRemoteDepth2(distDir))).difference(additionalAssets);
+  const expectedAssets = new Set(await loadExpectedAssets(version, line));
+
+  let caution = false;
+  let update = false;
+
+  // generate comparison lists
+  const stagingDistIntersection = stagingAssets.intersection(distAssets);
+  const stagingDistUnion = stagingAssets.union(distAssets);
+  let notInActual = expectedAssets.difference(stagingAssets);
+  let stagingNotInExpected = stagingAssets.difference(expectedAssets);
+  let distNotInExpected = distAssets.difference(expectedAssets);
+
+  console.log('... Checking R2 assets');
+  // No expected asset list available for this line
+  if (expectedAssets.size === 0) {
+    console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m  No expected asset list is available for ${line}, does one need to be created?`);
+    console.log(`    https://github.com/nodejs/build/tree/main/ansible/www-standalone/tools/promote/expected_assets/${line}`);
+    return;
+  }
+
+  console.log(`... Expecting a total of ${expectedAssets.size} assets for ${line}`);
+  console.log(`... ${stagingAssets.size} assets waiting in R2 staging`);
+
+   // what might be overwritten by promotion?
+   if (stagingDistIntersection.size) {
+    caution = true;
+    console.log(` \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m  ${stagingDistIntersection.size} assets already promoted in R2 will be overwritten, is this OK?`);
+    if (stagingDistIntersection.size <= 10) {
+      stagingDistIntersection.forEach((a) => console.log(`    • ${a}`));
+    }
+  } else {
+    console.log(`... ${distAssets.size} assets already promoted in R2`);
+  }
+
+  if (!notInActual.size) { // perfect staging state, we have everything we need
+    console.log(` \u001b[32m\u001b[1m✓\u001b[22m\u001b[39m  Complete set of expected assets in place for ${line}`);
+  } else { // missing some assets and they're not in staging, are you impatient?
+    caution = true;
+    console.log(` \u001b[33m\u001b[1m⚠\u001b[22m\u001b[39m  The following assets are expected for ${line} but are currently missing from R2 staging:`);
+    notInActual.forEach((a) => console.log(`    • ${a}`));
+  }
+
+  // bogus unexpected files found in staging, not good
+  if (stagingNotInExpected.size) {
+    caution = true;
+    update = true;
+    console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m  The following assets were found in R2 staging but are not expected for ${line}:`);
+    stagingNotInExpected.forEach((a) => console.log(`    • ${a}`));
+  }
+
+  // bogus unexpected files found in dist, not good
+  if (distNotInExpected.size) {
+    caution = true;
+    update = true;
+    console.log(` \u001b[31m\u001b[1m✖\u001b[22m\u001b[39m  The following assets were already promoted in R2 but are not expected for ${line}:`);
+    distNotInExpected.forEach((a) => console.log(`    • ${a}`));
+  }
+
+  // do we need to provide final notices?
+  if (update) {
+    console.log(`    Does the expected assets list for ${line} need to be updated?`);
+    console.log(`    https://github.com/nodejs/build/tree/main/ansible/www-standalone/tools/promote/expected_assets/${line}`);
+  }
+  if (caution) {
+    console.log('    \u001b[33mPromote if you are certain this is the the correct course of action\u001b[39m');
+  }
+}
+
+function versionToLine (version) {
+  return version.replace(/^(v\d+)\.[\d.]+.*/g, '$1.x');
+}
+
+export { checkArgs, run };

ansible/www-standalone/tools/promote/promote_release.sh

@@ -24,6 +24,11 @@ dirmatch=$release_dirmatch
 
 node --no-warnings /home/staging/tools/promote/check_assets.js $srcdir/$2 $dstdir/$2
 
+relative_srcdir=${srcdir/$staging_rootdir/"$site/"}
+relative_dstdir=${dstdir/$dist_rootdir/"$site/"}
+
+node --no-warnings /home/staging/tools/promote/check_r2_assets.mjs $staging_bucket/$relative_srcdir/$2 $prod_bucket/$relative_dstdir/$2
+
 while true; do
   echo -n "Are you sure you want to promote the $2 assets? [y/n] "
   yorn=""