Update node, 15.14.0 -> 16.20.0 (#1232)

This allows us to use the native ARM64 version on MacOS.
Also update the test scripts to work on ARM64 mac, and skip tests that aren't relevant.

Author: dschuff

.circleci/config.yml

@@ -10,12 +10,39 @@ executors:
   mac:
     environment:
       EMSDK_NOTTY: "1"
-      # Without this, any `brew installl` command will result in self-update of
+      # Without this, any `brew install` command will result in self-update of
       # brew itself which takes more than 4 minutes.
       HOMEBREW_NO_AUTO_UPDATE: "1"
     macos:
       xcode: "12.5.1"
     resource_class: macos.x86.medium.gen2
+  mac_arm64:
+    environment:
+      EMSDK_NOTTY: "1"
+      # Without this, any `brew install` command will result in self-update of
+      # brew itself which takes more than 4 minutes.
+      HOMEBREW_NO_AUTO_UPDATE: "1"
+    macos:
+      xcode: "13.4.1"
+    resource_class: macos.m1.medium.gen1
+
+commands:
+  setup-macos:
+    steps:
+      - checkout
+      - run:
+          name: Install CMake
+          command: brew install cmake
+  test-macos:
+    steps:
+      - run:
+          name: test.sh
+          command: test/test.sh
+      - run:
+          name: test.py
+          command: |
+            source emsdk_env.sh
+            test/test.py
 
 jobs:
   flake8:
@@ -59,16 +86,13 @@ jobs:
   test-mac:
     executor: mac
     steps:
-      - checkout
-      - run:
-          name: Install cmake
-          command: brew install cmake
-      - run: test/test.sh
-      - run:
-          name: test.py
-          command: |
-            source emsdk_env.sh
-            test/test.py
+      - setup-macos
+      - test-macos
+  test-mac-arm64:
+    executor: mac_arm64
+    steps:
+      - setup-macos
+      - test-macos
   test-windows:
     executor:
       name: win/vs2019
@@ -236,6 +260,9 @@ workflows:
   test-mac:
     jobs:
       - test-mac
+  test-mac-arm64:
+    jobs:
+      - test-mac-arm64
   test-windows:
     jobs:
       - test-windows

docker/Dockerfile

@@ -62,7 +62,7 @@ COPY --from=stage_build /emsdk /emsdk
 # This will let use tools offered by this image inside other Docker images
 # (sub-stages) or with custom / no entrypoint
 ENV EMSDK=/emsdk \
-    PATH="/emsdk:/emsdk/upstream/emscripten:/emsdk/upstream/bin:/emsdk/node/15.14.0_64bit/bin:${PATH}"
+    PATH="/emsdk:/emsdk/upstream/emscripten:/emsdk/upstream/bin:/emsdk/node/16.20.0_64bit/bin:${PATH}"
 
 # ------------------------------------------------------------------------------
 # Create a 'standard` 1000:1000 user

emsdk_manifest.json

@@ -298,6 +298,54 @@
     "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%"
   },
 
+  {
+    "id": "node",
+    "version": "16.20.0",
+    "bitness": 32,
+    "arch": "x86",
+    "windows_url": "node-v16.20.0-win-x86.zip",
+    "activated_path": "%installation_dir%/bin",
+    "activated_path_skip": "node",
+    "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'",
+    "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%"
+  },
+  {
+    "id": "node",
+    "version": "16.20.0",
+    "arch": "arm",
+    "bitness": 32,
+    "linux_url": "node-v16.20.0-linux-armv7l.tar.xz",
+    "activated_path": "%installation_dir%/bin",
+    "activated_path_skip": "node",
+    "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'",
+    "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%"
+  },
+  {
+    "id": "node",
+    "version": "16.20.0",
+    "bitness": 64,
+    "arch": "x86_64",
+    "macos_url": "node-v16.20.0-darwin-x64.tar.gz",
+    "windows_url": "node-v16.20.0-win-x64.zip",
+    "linux_url": "node-v16.20.0-linux-x64.tar.xz",
+    "activated_path": "%installation_dir%/bin",
+    "activated_path_skip": "node",
+    "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'",
+    "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%"
+  },
+  {
+    "id": "node",
+    "version": "16.20.0",
+    "arch": "aarch64",
+    "bitness": 64,
+    "macos_url": "node-v16.20.0-darwin-arm64.tar.gz",
+    "linux_url": "node-v16.20.0-linux-arm64.tar.xz",
+    "activated_path": "%installation_dir%/bin",
+    "activated_path_skip": "node",
+    "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'",
+    "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%"
+  },
+
 
   {
     "id": "python",
@@ -602,19 +650,19 @@
   {
     "version": "main",
     "bitness": 64,
-    "uses": ["python-3.9.2-nuget-64bit", "llvm-git-main-64bit", "node-15.14.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
+    "uses": ["python-3.9.2-nuget-64bit", "llvm-git-main-64bit", "node-16.20.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
     "os": "win"
   },
   {
     "version": "main",
     "bitness": 64,
-    "uses": ["python-3.9.2-64bit", "llvm-git-main-64bit", "node-15.14.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
+    "uses": ["python-3.9.2-64bit", "llvm-git-main-64bit", "node-16.20.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
     "os": "macos"
   },
   {
     "version": "main",
     "bitness": 64,
-    "uses": ["llvm-git-main-64bit", "node-15.14.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
+    "uses": ["llvm-git-main-64bit", "node-16.20.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
     "os": "linux"
   },
   {
@@ -626,30 +674,30 @@
   {
     "version": "releases-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-15.14.0-64bit", "releases-%releases-tag%-64bit"],
+    "uses": ["node-16.20.0-64bit", "releases-%releases-tag%-64bit"],
     "os": "linux",
     "custom_install_script": "emscripten_npm_install"
   },
   {
     "version": "releases-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-15.14.0-64bit", "python-3.9.2-64bit", "releases-%releases-tag%-64bit"],
+    "uses": ["node-16.20.0-64bit", "python-3.9.2-64bit", "releases-%releases-tag%-64bit"],
     "os": "macos",
     "arch": "x86_64",
     "custom_install_script": "emscripten_npm_install"
   },
   {
     "version": "releases-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-15.14.0-64bit", "python-3.9.2-64bit", "releases-%releases-tag%-64bit"],
+    "uses": ["node-16.20.0-64bit", "python-3.9.2-64bit", "releases-%releases-tag%-64bit"],
     "os": "macos",
     "arch": "aarch64",
     "custom_install_script": "emscripten_npm_install"
   },
   {
     "version": "releases-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-15.14.0-64bit", "python-3.9.2-nuget-64bit", "java-8.152-64bit", "releases-%releases-tag%-64bit"],
+    "uses": ["node-16.20.0-64bit", "python-3.9.2-nuget-64bit", "java-8.152-64bit", "releases-%releases-tag%-64bit"],
     "os": "win",
     "custom_install_script": "emscripten_npm_install"
   }

scripts/update_node.py

@@ -16,14 +16,15 @@
 import os
 import shutil
 
-version = '15.14.0'
-base = 'https://nodejs.org/dist/latest-v15.x/'
+version = '16.20.0'
+base = 'https://nodejs.org/dist/latest-v16.x/'
 upload_base = 'gs://webassembly/emscripten-releases-builds/deps/'
 
 suffixes = [
     '-win-x86.zip',
     '-win-x64.zip',
     '-darwin-x64.tar.gz',
+    '-darwin-arm64.tar.gz',
     '-linux-x64.tar.xz',
     '-linux-arm64.tar.xz',
     '-linux-armv7l.tar.xz',

test/test.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import json
 import os
+import platform
 import shutil
 import subprocess
 import sys
@@ -9,6 +10,7 @@
 
 WINDOWS = sys.platform.startswith('win')
 MACOS = sys.platform == 'darwin'
+MACOS_ARM64 = MACOS and platform.machine() == 'arm64'
 
 emconfig = os.path.abspath('.emscripten')
 assert os.path.exists(emconfig)
@@ -197,13 +199,17 @@ def test_closure(self):
     check_call(upstream_emcc + ' hello_world.c --closure=1')
 
   def test_specific_version(self):
+    if MACOS_ARM64:
+      self.skipTest('Old sdk versions do not have ARM64 binaries')
     print('test specific release (new, short name)')
     run_emsdk('install 1.38.33')
     print('another install, but no need for re-download')
     checked_call_with_output(emsdk + ' install 1.38.33', expected='Skipped', unexpected='Downloading:')
     run_emsdk('activate 1.38.33')
 
   def test_specific_version_full(self):
+    if MACOS_ARM64:
+      self.skipTest('Old sdk versions do not have ARM64 binaries')
     print('test specific release (new, full name)')
     run_emsdk('install sdk-1.38.33-64bit')
     run_emsdk('activate sdk-1.38.33-64bit')
@@ -220,7 +226,8 @@ def test_binaryen_from_source(self):
   def test_no_32bit(self):
     print('test 32-bit error')
     emsdk_hacked = hack_emsdk('not is_os_64bit()', 'True')
-    failing_call_with_output('python %s install latest' % emsdk_hacked, 'this tool is only provided for 64-bit OSes')
+    failing_call_with_output('%s %s install latest' % (sys.executable, emsdk_hacked),
+                             'this tool is only provided for 64-bit OSes')
     os.remove(emsdk_hacked)
 
   def test_update_no_git(self):
@@ -243,10 +250,10 @@ def test_update_no_git(self):
 
   def test_install_arbitrary(self):
     # Test that its possible to install arbrary emscripten-releases SDKs
-    run_emsdk('install 5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2')
+    run_emsdk('install 1b7f7bc6002a3ca73647f41fc10e1fac7f06f804')
 
     # Check that its not re-downloaded
-    checked_call_with_output(emsdk + ' install 5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2', expected='Skipped', unexpected='Downloading:')
+    checked_call_with_output(emsdk + ' install 1b7f7bc6002a3ca73647f41fc10e1fac7f06f804', expected='Skipped', unexpected='Downloading:')
 
   def test_install_tool(self):
     # Test that its possible to install emscripten as tool instead of SDK

test/test.sh

@@ -7,20 +7,22 @@ set -e
 
 # Test that arbitrary (non-released) versions can be installed and
 # activated.
-./emsdk install sdk-upstream-5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2
-./emsdk activate sdk-upstream-5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2
+./emsdk install sdk-upstream-1b7f7bc6002a3ca73647f41fc10e1fac7f06f804
+./emsdk activate sdk-upstream-1b7f7bc6002a3ca73647f41fc10e1fac7f06f804
 source ./emsdk_env.sh
 which emcc
 emcc -v
 
-# Install an older version of the SDK that requires EM_CACHE to be
-# set in the environment, so that we can test it is later removed
-./emsdk install sdk-1.39.15
-./emsdk activate sdk-1.39.15
-source ./emsdk_env.sh
-which emcc
-emcc -v
-test -n "$EM_CACHE"
+if [[ $(uname -m) == "x86_64" ]]; then
+  # Install an older version of the SDK that requires EM_CACHE to be
+  # set in the environment, so that we can test it is later removed
+  ./emsdk install sdk-1.39.15
+  ./emsdk activate sdk-1.39.15
+  source ./emsdk_env.sh
+  which emcc
+  emcc -v
+  test -n "$EM_CACHE"
+fi
 
 # Install the latest version of the SDK which is the expected precondition
 # of test.py.