Docker image with prebuilt pyodide (pyodide#787)

Author: MischaPanch

.dockerignore

@@ -1,4 +1,4 @@
 emsdk/emsdk/
 cpython/installs/
 cpython/build/
-packages/
+packages/*/build/

.github/workflows/prebuilt_docker_image.yaml

@@ -0,0 +1,39 @@
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version of the docker image to build.
+          The same version of the pyodide-env image must already exist'
+        required: true
+
+jobs:
+  build_image:
+    runs-on: ubuntu-latest
+#   This is the recommended use of github's docker build-push action
+#   See https://github.com/marketplace/actions/build-and-push-docker-images#git-context
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          file: ./Dockerfile-prebuilt
+          push: true
+          build-args: |
+            VERSION=${{ github.event.inputs.version }}
+          tags: "iodide/pyodide:${{ github.event.inputs.version }}"
+      -
+        name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

Dockerfile-prebuilt

@@ -0,0 +1,14 @@
+ARG VERSION
+
+FROM iodide/pyodide-env:${VERSION}
+
+USER root
+
+ENV EMSDK_NUM_CORES=4 EMCC_CORES=4 PYODIDE_JOBS=4
+
+COPY . pyodide
+
+# the rm at the end deletes the build results such that the resulting image can still be used for building pyodide
+# from source (including partial and customized builds). Due to the previous run of make, builds
+# executed with this image will be much faster than the ones executed with pyodide-env
+RUN cd pyodide && make && rm -r pyodide/build

docs/building_from_sources.md

@@ -45,16 +45,21 @@ make
 ## Using Docker
 
 We provide a Debian-based Docker image on Docker Hub with the dependencies
-already installed to make it easier to build Pyodide. Note that building from
-the Docker image is *very* slow on Mac, building on the host machine is
-preferred if at all possible.
+already installed to make it easier to build Pyodide. On top of that we provide a
+pre-built image which can be used for fast custom and partial builds of pyodide.
+Note that building from the non pre-built the Docker image is *very* slow on Mac,
+building on the host machine is preferred if at all possible.
 
 1. Install Docker
 
-2. From a git checkout of Pyodide, run `./run_docker`
+2. From a git checkout of Pyodide, run `./run_docker` or `./run_docker --pre-built`
 
 3. Run `make` to build.
 
+Note: You can control the resources allocated to the build by setting the env vars
+`EMSDK_NUM_CORE`, `EMCC_CORES` and `PYODIDE_JOBS` (the default for each is 4).
+
+
 If running ``make`` deterministically stops at one point in each subsequent try, increasing
 the maximum RAM usage available to the docker container might help [This is different
 from the physical RAM capacity inside the system]. Ideally, at least 3 GB of RAM

run_docker

@@ -1,13 +1,77 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+
+function usage() {
+  cat > /dev/stdout <<EOF
+Usage:
+  run_docker [OPTIONS]
+
+  Optional flags:
+    -h, --help                  Show this information and exit
+    --pre-built                 Use the prebuilt pyodide image.
+                                This is ignored if the env var PYODIDE_DOCKER_IMAGE is set
+    -p, --port <port>           System port to which to forward.
+                                This is ignored if the env var PYODIDE_SYSTEM_PORT is set
+
+  Prerequisites:
+    Docker has to be set up on your system
+EOF
+}
+
+function error() {
+  usage
+  exit 255
+}
+
+
+PYODIDE_IMAGE_TAG="0.16.1"
+DEFAULT_PYODIDE_DOCKER_IMAGE="iodide/pyodide-env:${PYODIDE_IMAGE_TAG}"
+DEFAULT_PYODIDE_SYSTEM_PORT="8000"
+
+while [[ $# -gt 0 ]]
+do
+  key="$1"
+  case $key in
+      -h|--help)
+        usage
+        exit 0
+      ;;
+      --pre-built)
+        if [[ -n ${PYODIDE_DOCKER_IMAGE} ]]; then
+          echo "WARNING: will use the env var PYODIDE_DOCKER_IMAGE=${PYODIDE_DOCKER_IMAGE},
+          the flag --pre-built has no effect"
+        fi
+        DEFAULT_PYODIDE_DOCKER_IMAGE="iodide/pyodide:${PYODIDE_IMAGE_TAG}"
+        shift
+      ;;
+      -p|--port)
+        if [ "$#" -lt 2 ]; then
+          >&2 echo "port cannot be empty"
+          error
+        fi
+        if [[ -n ${PYODIDE_SYSTEM_PORT} ]]; then
+          echo "WARNING: will use the env var PYODIDE_SYSTEM_PORT=${PYODIDE_SYSTEM_PORT} instead of the provided port"
+        fi
+        DEFAULT_PYODIDE_SYSTEM_PORT=$2
+        shift 2
+      ;;
+      *)
+        >&2 echo "Unknown option $1"
+        error
+      ;;
+  esac
+done
 
 PYODIDE_DOCKER_PORT=${PYODIDE_DOCKER_PORT:-"8000"}
-PYODIDE_SYSTEM_PORT=${PYODIDE_SYSTEM_PORT:-"8000"}
-PYODIDE_DOCKER_IMAGE=${PYODIDE_DOCKER_IMAGE:-"iodide/pyodide-env:0.16.1"}
+PYODIDE_SYSTEM_PORT=${PYODIDE_SYSTEM_PORT:-${DEFAULT_PYODIDE_SYSTEM_PORT}}
+PYODIDE_DOCKER_IMAGE=${PYODIDE_DOCKER_IMAGE:-${DEFAULT_PYODIDE_DOCKER_IMAGE}}
 
 exec docker run \
-    -p $PYODIDE_SYSTEM_PORT:$PYODIDE_DOCKER_PORT \
+    -p "$PYODIDE_SYSTEM_PORT":"$PYODIDE_DOCKER_PORT" \
     -it --rm \
     -v $PWD:/src \
     --user root -e NB_UID=$UID -e NB_GID=$GID \
-    ${PYODIDE_DOCKER_IMAGE} \
+    "${PYODIDE_DOCKER_IMAGE}" \
     /bin/bash

setup.cfg

@@ -4,7 +4,7 @@ addopts =
     -r sxX
 
 [bumpversion]
-current_version = 0.15.0
+current_version = 0.16.1
 commit = True
 tag = True
 tag_name = {new_version}

test/pyodide_build/test_run_docker.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+import subprocess
+
+BASE_DIR = Path(__file__).parents[2]
+
+
+def test_run_docker_script():
+    res = subprocess.run(
+        ["bash", str(BASE_DIR / "run_docker"), "--help"],
+        check=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    assert "Usage:\n  run_docker" in res.stdout.decode("utf-8")
+
+    res = subprocess.run(
+        ["bash", str(BASE_DIR / "run_docker"), "--invalid-param"],
+        check=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    assert res.returncode > 0
+    assert "Unknown option --invalid-param" in res.stderr.decode("utf-8")