[3.11] gh-95853: Add script to automate WASM build (GH-95828, GH-95985, GH-96045, GH-96389, GH-96744) (GH-96749)

Automate WASM build with a new Python script. The script provides
several build profiles with configure flags for Emscripten flavors
and WASI. The script can detect and use Emscripten SDK and WASI SDK from
default locations or env vars.

``configure`` now detects Node arguments and creates HOSTRUNNER
arguments for Node 16. It also sets correct arguments for
``wasm64-emscripten``.

Author: tiran

Lib/distutils/tests/test_sysconfig.py

@@ -49,6 +49,7 @@ def test_get_config_vars(self):
         self.assertIsInstance(cvars, dict)
         self.assertTrue(cvars)
 
+    @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
     def test_srcdir(self):
         # See Issues #15322, #15364.
         srcdir = sysconfig.get_config_var('srcdir')

Lib/test/test_decimal.py

@@ -37,7 +37,7 @@
                           requires_legacy_unicode_capi, check_sanitizer)
 from test.support import (TestFailed,
                           run_with_locale, cpython_only,
-                          darwin_malloc_err_warning)
+                          darwin_malloc_err_warning, is_emscripten)
 from test.support.import_helper import import_fresh_module
 from test.support import threading_helper
 from test.support import warnings_helper
@@ -5623,6 +5623,7 @@ def __abs__(self):
     # Issue 41540:
     @unittest.skipIf(sys.platform.startswith("aix"),
                      "AIX: default ulimit: test is flaky because of extreme over-allocation")
+    @unittest.skipIf(is_emscripten, "Test is unstable on Emscripten")
     @unittest.skipIf(check_sanitizer(address=True, memory=True),
                      "ASAN/MSAN sanitizer defaults to crashing "
                      "instead of returning NULL for malloc failure.")

Lib/test/test_sysconfig.py

@@ -439,6 +439,7 @@ def test_platform_in_subprocess(self):
         self.assertEqual(status, 0)
         self.assertEqual(my_platform, test_platform)
 
+    @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds")
     def test_srcdir(self):
         # See Issues #15322, #15364.
         srcdir = sysconfig.get_config_var('srcdir')

Lib/test/test_unicode_file_functions.py

@@ -6,6 +6,7 @@
 import warnings
 from unicodedata import normalize
 from test.support import os_helper
+from test import support
 
 
 filenames = [
@@ -123,6 +124,10 @@ def test_open(self):
     # NFKD in Python is useless, because darwin will normalize it later and so
     # open(), os.stat(), etc. don't raise any exception.
     @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
+    @unittest.skipIf(
+        support.is_emscripten or support.is_wasi,
+        "test fails on Emscripten/WASI when host platform is macOS."
+    )
     def test_normalize(self):
         files = set(self.files)
         others = set()

Lib/test/test_warnings/__init__.py

@@ -489,7 +489,14 @@ def test_warn_explicit_non_ascii_filename(self):
                 module=self.module) as w:
             self.module.resetwarnings()
             self.module.filterwarnings("always", category=UserWarning)
-            for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"):
+            filenames = ["nonascii\xe9\u20ac"]
+            if not support.is_emscripten:
+                # JavaScript does not like surrogates.
+                # Invalid UTF-8 leading byte 0x80 encountered when
+                # deserializing a UTF-8 string in wasm memory to a JS
+                # string!
+                filenames.append("surrogate\udc80")
+            for filename in filenames:
                 try:
                     os.fsencode(filename)
                 except UnicodeEncodeError:

Makefile.pre.in

@@ -817,10 +817,11 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 # wasm assets directory is relative to current build dir, e.g. "./usr/local".
 # --preload-file turns a relative asset path into an absolute path.
 
+.PHONY: wasm_stdlib
+wasm_stdlib: $(WASM_STDLIB)
 $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
 	    $(srcdir)/Tools/wasm/wasm_assets.py \
-	    Makefile pybuilddir.txt Modules/Setup.local \
-	    python.html python.worker.js
+	    Makefile pybuilddir.txt Modules/Setup.local
 	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
 	    --buildroot . --prefix $(prefix)
 
@@ -1713,6 +1714,10 @@ buildbottest: all
 		fi
 		$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
+# Like testall, but run Python tests with HOSTRUNNER directly.
+hostrunnertest: all
+	$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test -u all $(TESTOPTS)
+
 pythoninfo: all
 		$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo
 

Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst

@@ -0,0 +1,2 @@
+The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and
+test steps for building CPython on WebAssembly platforms.

Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst

@@ -0,0 +1,2 @@
+The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for
+broken EMSDK versions, and warns about pkg-config env vars.

Modules/pyexpat.c

@@ -775,7 +775,7 @@ readinst(char *buf, int buf_size, PyObject *meth)
     Py_ssize_t len;
     const char *ptr;
 
-    str = PyObject_CallFunction(meth, "n", buf_size);
+    str = PyObject_CallFunction(meth, "i", buf_size);
     if (str == NULL)
         goto error;
 

Python/sysmodule.c

@@ -2775,14 +2775,18 @@ EM_JS(char *, _Py_emscripten_runtime, (void), {
     if (typeof navigator == 'object') {
         info = navigator.userAgent;
     } else if (typeof process == 'object') {
-        info = "Node.js ".concat(process.version)
+        info = "Node.js ".concat(process.version);
     } else {
-        info = "UNKNOWN"
+        info = "UNKNOWN";
     }
     var len = lengthBytesUTF8(info) + 1;
     var res = _malloc(len);
-    stringToUTF8(info, res, len);
+    if (res) stringToUTF8(info, res, len);
+#if __wasm64__
+    return BigInt(res);
+#else
     return res;
+#endif
 });
 
 static PyObject *