Use rollup (pyodide#1575)

Co-authored-by: Roman Yurchak <[email protected]>

Author: Hood Chatham

Makefile

@@ -42,7 +42,7 @@ build/pyodide.asm.js: \
 	src/pystone.py \
 	src/_testcapi.py \
 	src/webbrowser.py \
-	$(wildcard src/pyodide-py/pyodide/*.py) \
+	$(wildcard src/py/pyodide/*.py) \
 	$(CPYTHONLIB)
 	date +"[%F %T] Building pyodide.asm.js..."
 	[ -d build ] || mkdir build
@@ -52,8 +52,8 @@ build/pyodide.asm.js: \
 		--preload-file src/webbrowser.py@/lib/python$(PYMINOR)/webbrowser.py \
 		--preload-file src/_testcapi.py@/lib/python$(PYMINOR)/_testcapi.py \
 		--preload-file src/pystone.py@/lib/python$(PYMINOR)/pystone.py \
-		--preload-file src/pyodide-py/pyodide@/lib/python$(PYMINOR)/site-packages/pyodide \
-		--preload-file src/pyodide-py/_pyodide@/lib/python$(PYMINOR)/site-packages/_pyodide \
+		--preload-file src/py/pyodide@/lib/python$(PYMINOR)/site-packages/pyodide \
+		--preload-file src/py/_pyodide@/lib/python$(PYMINOR)/site-packages/_pyodide \
 		--exclude-file "*__pycache__*" \
 		--exclude-file "*/test/*"		\
 		--exclude-file "*/tests/*" \
@@ -78,10 +78,9 @@ env:
 
 
 .PHONY: build/pyodide.js
-build/pyodide.js: src/pyodide.js
-	cp $< $@
-	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@
-
+build/pyodide.js: src/js/*.js
+	npm install ./src/js
+	npx rollup -c src/js/rollup.config.js
 
 build/test.html: src/templates/test.html
 	cp $< $@

docs/conf.py

@@ -14,7 +14,7 @@
     str(base_dir),
     str(base_dir / "pyodide-build"),
     str(base_dir / "docs/sphinx_pyodide"),
-    str(base_dir / "src/pyodide-py"),
+    str(base_dir / "src/py"),
     str(base_dir / "packages/micropip/micropip"),
 ]
 sys.path = path_dirs + sys.path

docs/development/maintainers.md

@@ -14,7 +14,7 @@ the latest release branch named `stable` (due to ReadTheDocs constraints).
    `https://cdn.jsdelivr.net/pyodide/dev/full/` in `./docs/` replace `dev` with
    the release version `vX.Y.Z` (note the presence of the leading `v`). This
    also applies to `docs/conf.py`
-2. Set version in `src/pyodide-py/pyodide/__init__.py`
+2. Set version in `src/py/pyodide/__init__.py`
 3. Make sure the change log is up to date.
     - Indicate the release date in the change log.
     - Generate the list of contributors for the release at the end of the
@@ -53,7 +53,7 @@ the latest release branch named `stable` (due to ReadTheDocs constraints).
    ```
    where `BB` is the last version of the `pyodide-env` Docker image.
 7. Revert Step 1. and increment the version in
-   `src/pyodide-py/pyodide/__init__.py` to the next version specified by
+   `src/py/pyodide/__init__.py` to the next version specified by
    Semantic Versioning.
 
 ### Making a minor release

repository-structure.md

@@ -0,0 +1,23 @@
+The Pyodide runtime consists of the following components, sorted in terms of
+initialization-time (or import-time) dependencies.
+
+1. CPython
+2. The py/_pyodide package which is a Python package with pure Python code
+   avaiable in the inner stage of the Pyodide bootstrap process.
+3. The core/pyodide code, implemented in a mix of C and Javascript, which embeds
+   the CPython interpreter in an emscripten application. This relies on
+   py/pyodide and js/pyodide at runtime. The final stage of initialization is to
+   import py/pyodide.
+4. The py/pyodide package which has Python code that is needed for the outer
+   stage of the Pyodide bootstrap process. py/pyodide relies on core/pyodide at
+   import time and relies on js/pyodide at runtime.
+5. The js/pyodide package which defines the Javascript public API, sets up the
+   process of loading the core/pyodide emscripten application + CPython
+   interpreter, and then completes the bootstrap by injecting the js/pyodide
+   API into the Python `sys.modules`.
+6. The packages directory, which contains a large number of CPython packages
+   built to run in Pyodide.
+
+One of our long-term organizational goals is to carefully organize core/pyodide,
+py/_pyodide, py/pyodide, and js/pyodide to clarify which functionality is not part of
+runtime dependencies that define Pyodide's core behavior.

src/core/README.md

@@ -0,0 +1,33 @@
+# core/pyodide
+
+The C and Javascript code in this package is responsible for embedding the
+Python interpreter in our emscripten js/wasm application. The primary purpose of
+this code is to define the foreign function interface between Python and
+Javascript. Once this foreign function interface is defined, more complex
+behaviors are better defined in Python when possible for easier development and
+debugging.
+
+In particular, when possible logic should be moved from core/pyodide to
+py/_pyodide or to py/pyodide.
+
+The core/pyodide code is responsible for the following main steps:
+
+1. Initialize the CPython interpreter
+2. Import py/_pyodide
+3. Initialize `_pyodide_core` which is a Python C extension that we use to make
+   functionality available to py/pyodide.
+4. Set up functionality to automatically convert functions from Javascript to
+   CPython calling conventions (`error_handling.h`).
+5. Set up the "hiwire" side table to hold references to Javascript objects --
+   necessary because wasm variables can only hold numbers (`hiwire.c`).
+6. Set up type conversions of basic types between Python and Javascript
+   (`js2python.c` and `python2js.c`).
+7. Set up Proxying of remaining types between Python and Javascript (`jsproxy.c`
+   and `pyproxy.c`). This is the most complicated part of the Pyodide runtime
+   and involves careful conversion between [abstract Javascript object
+   protocols](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy)
+   (see also [Javascript Iteration
+   Protocols](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols)
+   )
+   and [Python object protocols](https://docs.python.org/3/c-api/abstract.html).
+8. Add `_pyodide_core` to `sys.modules` and import py/pyodide.

src/core/main.c

@@ -116,6 +116,12 @@ main(int argc, char** argv)
     FATAL_ERROR("JsRef doesn't have the same size as int.");
   }
 
+  PyObject* _pyodide = PyImport_ImportModule("_pyodide");
+  if (_pyodide == NULL) {
+    FATAL_ERROR("Failed to import pyodide module");
+  }
+  Py_CLEAR(_pyodide);
+
   PyObject* core_module = NULL;
   core_module = PyModule_Create(&core_module_def);
   if (core_module == NULL) {

src/js/README.md

@@ -0,0 +1,12 @@
+# js/pyodide
+
+The Javascript code in this folder is responsible for the following tasks:
+
+1. Defines the public Javascript API
+    - Package loading code to allow loading of other Python packages.
+    - Can load micropip.py to bootstrap loading of pure Python wheels
+2. Loads the CPython interpreter and the core/pyodide emscripten application
+   which embeds the interpreter.
+3. Injects the js/pyodide Javascript API into sys.modules. This is the final
+   runtime dependency for core/pyodide & py/pyodide, so after this step the
+   interpreter is fully up and running.

src/js/package-lock.json

@@ -0,0 +1,8 @@
+{
+  "name" : "pyodide",
+  "version": "0.18.0dev0",
+  "dependencies": {
+    "prettier": "^2.2.1",
+    "rollup-plugin-terser": "^7.0.2"
+  }
+}

src/js/package.json

@@ -38,7 +38,10 @@ globalThis.loadPyodide = async function (config = {}) {
   // Note: PYODIDE_BASE_URL is an environment variable replaced in
   // in this template in the Makefile. It's recommended to always set
   // indexURL in any case.
-  let baseURL = config.indexURL || "{{ PYODIDE_BASE_URL }}";
+  if (!config.indexURL) {
+    throw new Error("Please provide indexURL parameter to loadPyodide");
+  }
+  let baseURL = config.indexURL;
   if (baseURL.endsWith(".js")) {
     baseURL = baseURL.substr(0, baseURL.lastIndexOf("/"));
   }

src/pyodide.js src/js/pyodide.js

@@ -0,0 +1,31 @@
+import { terser } from "rollup-plugin-terser";
+
+function config({ input, format, minify, ext = "js" }) {
+  const dir = `build/`;
+  // const minifierSuffix = minify ? ".min" : "";
+  const minifierSuffix = "";
+  return {
+    input: `./src/js/${input}.js`,
+    output: {
+      name: "loadPyodide",
+      file: `${dir}/${input}${minifierSuffix}.${ext}`,
+      format,
+      sourcemap: true,
+    },
+    plugins: [
+      minify
+        ? terser({
+            compress: true,
+            mangle: false,
+          })
+        : undefined,
+    ].filter(Boolean),
+  };
+}
+
+export default [
+  // { input: "pyodide", format: "esm", minify: false, ext: "mjs" },
+  { input: "pyodide", format: "esm", minify: true, ext: "mjs" },
+  // { input: "pyodide", format: "umd", minify: false },
+  { input: "pyodide", format: "umd", minify: true },
+].map(config);