Merge branch 'main' into feature/0.18.0-nano

Author: phorward

.circleci/config.yml

@@ -1,9 +1,9 @@
-version: 2
+version: 2.1
 
 defaults: &defaults
   working_directory: ~/repo
   docker:
-    - image: pyodide/pyodide-env:15
+    - image: pyodide/pyodide-env:19
   environment:
     - EMSDK_NUM_CORES: 3
       EMCC_CORES: 3
@@ -21,25 +21,18 @@ jobs:
           command: make lint
 
   test-docs:
-    working_directory: ~/repo
+    <<: *defaults
     resource_class: small
-
-    docker:
-      - image: cimg/python:3.8.2-node
-
     steps:
       - checkout
 
-      - run:
-          name: Install prerequisites
-          command: |
-            pip install -r docs/requirements-doc.txt
-            pip install pytest
-            sudo npm install -g jsdoc
-
       - run:
           name: Test docs
-          command: pytest docs/sphinx_pyodide/tests
+          command: |
+            mkdir test-results
+            pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml
+      - store_test_results:
+          path: test-results
 
   build-core:
     <<: *defaults
@@ -72,7 +65,7 @@ jobs:
           command: |
             ccache -z
             # The following packages are currently used in the main pyodide test suite
-            PYODIDE_PACKAGES="micropip,pyparsing,pytz,packaging,kiwisolver" make
+            PYODIDE_PACKAGES="micropip,pyparsing,pytz,packaging,Jinja2" make
             ccache -s
 
       - run:
@@ -116,7 +109,7 @@ jobs:
           no_output_timeout: 1800
           command: |
             ccache -z
-            make
+            make -C packages
             ccache -s
 
       - run:
@@ -137,65 +130,83 @@ jobs:
       - store_artifacts:
           path: /root/repo/build/
 
-  test-core-firefox:
+      - store_artifacts:
+          path: /root/repo/packages/build-logs
+
+  test-main:
+    parameters:
+      test-params:
+        description: The tests to run.
+        type: string
     <<: *defaults
     steps:
       - attach_workspace:
           at: .
       - run:
           name: test
           command: |
-            tools/pytest_wrapper.py src packages/micropip/ -v -k firefox -n 3
-
-  test-core-chrome:
+            mkdir test-results
+            tools/pytest_wrapper.py \
+              --junitxml=test-results/junit.xml \
+              --verbose \
+              << parameters.test-params >>
+      - store_test_results:
+          path: test-results
+
+  benchmark-stack-size:
     <<: *defaults
     steps:
       - attach_workspace:
           at: .
       - run:
-          name: test
+          name: stack-size
           command: |
-            tools/pytest_wrapper.py src packages/micropip/ -v -k chrome
+            pytest -s benchmark/stack_usage.py  | sed -n 's/## //pg' || true
 
-  test-packages-firefox:
+  test-emsdk:
     <<: *defaults
+    resource_class: small
     steps:
       - attach_workspace:
           at: .
       - run:
           name: test
-          command: |
-            tools/pytest_wrapper.py packages/test* packages/*/test* -v -k firefox
+          command: make -C emsdk test
 
-  test-packages-chrome:
+  test-python:
+    # Unlike test-main this uses "resource_class small" and doesn't use "pytest_wrapper."
     <<: *defaults
+    resource_class: small
     steps:
-      - attach_workspace:
-          at: .
+      - checkout
       - run:
           name: test
           command: |
-            tools/pytest_wrapper.py packages/test* packages/*/test* -v -k chrome -n 2
-
-  test-emsdk:
+            mkdir test-results
+            pytest \
+              --junitxml=test-results/junit.xml \
+              --verbose \
+              -k 'not (chrome or firefox or node)' \
+              --cov=pyodide_build --cov=pyodide \
+              src pyodide-build packages/micropip/
+
+      - store_test_results:
+          path: test-results
+
+  test-js:
     <<: *defaults
     resource_class: small
     steps:
       - attach_workspace:
           at: .
-      - run:
-          name: test
-          command: make -C emsdk test
-
-  test-python:
-    <<: *defaults
-    resource_class: small
-    steps:
-      - checkout
       - run:
           name: test
           command: |
-            pytest src pyodide_build packages/micropip/ -v -k 'not (chrome or firefox)' --cov=pyodide_build --cov=pyodide
+            cd src/js
+            npx tsd
+            npm install
+            npm test
+
 
   benchmark:
     <<: *defaults
@@ -211,6 +222,8 @@ jobs:
           path: /root/repo/build/benchmarks.json
 
   deploy-release:
+    # To reduce chance of deployment issues, try to keep the steps here as
+    # similar as possible to the steps in deploy-dev!
     resource_class: small
 
     docker:
@@ -223,7 +236,7 @@ jobs:
       - run:
           name: Install requirements
           command: |
-            apk add --no-cache --update python3
+            apk add --no-cache --update python3 make
             python3 -m pip install awscli
       - run:
           name: Deploy Github Releases
@@ -245,25 +258,26 @@ jobs:
             # update 301 redirect for the /latest/* route.
             aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json
 
-  deploy-s3:
+  deploy-dev:
+    # To reduce chance of deployment issues, try to keep the steps here as
+    # similar as possible to the steps in deploy-release!
     resource_class: small
 
     docker:
-      - image: circleci/python:3.7.7
+      - image: cibuilds/github:0.13
 
     steps:
       - checkout
       - attach_workspace:
           at: .
-      - run:
-          name: Set PYODIDE_BASE_URL
-          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make update_base_url
       - run:
           name: Install requirements
           command: |
-            sudo apt-get update
-            sudo apt-get install -y groff
+            apk add --no-cache --update python3 make
             python3 -m pip install awscli
+      - run:
+          name: Set PYODIDE_BASE_URL
+          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make update_base_url
       - run:
           name: Deploy to pyodide-cdn2.iodide.io
           command: |
@@ -278,50 +292,50 @@ workflows:
     jobs:
       - lint
       - test-docs
-      - build-core:
-          filters:
-            tags:
-              only: /.*/
+      - test-python
+      - build-core
       - build-packages:
           requires:
             - build-core
-          filters:
-            tags:
-              only: /.*/
-      - test-core-chrome:
+      - test-main:
+          name: test-core-chrome
+          test-params: -k chrome src packages/micropip
           requires:
             - build-core
-          filters:
-            tags:
-              only: /.*/
-      - test-core-firefox:
+      - test-main:
+          name: test-core-firefox
+          test-params: -k firefox src packages/micropip
           requires:
             - build-core
-          filters:
-            tags:
-              only: /.*/
-      - test-packages-chrome:
+      - test-main:
+          name: test-core-node
+          test-params: -k node src packages/micropip
+          requires:
+            - build-core
+      - test-main:
+          name: test-packages-chrome
+          test-params: -k chrome packages/test* packages/*/test*
           requires:
             - build-packages
-          filters:
-            tags:
-              only: /.*/
-      - test-packages-firefox:
+      - test-main:
+          name: test-packages-firefox
+          test-params: -k firefox packages/test* packages/*/test*
+          requires:
+            - build-packages
+      - test-main:
+          name: test-packages-node
+          test-params: -k "node and not numpy" packages/test* packages/*/test*
           requires:
             - build-packages
-          filters:
-            tags:
-              only: /.*/
       - test-emsdk:
           requires:
             - build-core
-          filters:
-            tags:
-              only: /.*/
-      - test-python:
-          filters:
-            tags:
-              only: /.*/
+      - test-js:
+          requires:
+            - build-core
+      - benchmark-stack-size:
+          requires:
+            - build-core
       - benchmark:
           requires:
             - build-packages
@@ -337,7 +351,7 @@ workflows:
               ignore: /.*/
             tags:
               only: /^\d+\.\d+\.\w+$/
-      - deploy-s3:
+      - deploy-dev:
           requires:
             - lint
             - test-python

.gitignore

@@ -1,6 +1,7 @@
 *.a
 *.o
 *.pyc
+src/js/pyproxy.gen.js
 
 *.egg-info/
 
@@ -18,9 +19,12 @@ build
 ccache
 cpython/downloads
 cpython/installs
+cpython/build.log
 docs/_build/
 emsdk/emsdk
 geckodriver.log
-node_modules/
+node_modules
 packages/.artifacts
 packages/*/build.log
+packages/build-logs
+dist/

.pre-commit-config.yaml

@@ -1,4 +1,6 @@
 exclude: '^.*patches|.*\.cgi$'
+default_language_version:
+    python : "3.9"
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.3.0