BLD Create an installable pyodide-build package (pyodide#1566)

Author: rth

.circleci/config.yml

@@ -189,7 +189,7 @@ jobs:
       - run:
           name: test
           command: |
-            pytest src pyodide_build packages/micropip/ -v -k 'not (chrome or firefox)' --cov=pyodide_build --cov=pyodide
+            pytest src pyodide-build packages/micropip/ -v -k 'not (chrome or firefox)' --cov=pyodide_build --cov=pyodide
 
   benchmark:
     <<: *defaults

.gitignore

@@ -24,3 +24,4 @@ geckodriver.log
 node_modules/
 packages/.artifacts
 packages/*/build.log
+dist/

Makefile

@@ -117,15 +117,15 @@ update_base_url: \
 	build/webworker.js
 
 test: all
-	pytest src emsdk/tests packages/*/test* pyodide_build -v
+	pytest src emsdk/tests packages/*/test* pyodide-build -v
 
 lint:
 	# check for unused imports, the rest is done by black
-	flake8 --select=F401 src tools pyodide_build benchmark conftest.py docs
+	flake8 --select=F401 src tools pyodide-build benchmark conftest.py docs
 	clang-format-6.0 -output-replacements-xml `find src -type f -regex ".*\.\(c\|h\\)"` | (! grep '<replacement ')
 	$(PRETTIER) --check `find src -type f -name '*.js'`
 	black --check .
-	mypy --ignore-missing-imports pyodide_build/ src/ packages/micropip/micropip/ packages/*/test* conftest.py docs
+	mypy --ignore-missing-imports pyodide-build/pyodide_build/ src/ packages/micropip/micropip/ packages/*/test* conftest.py docs
 
 apply-lint:
 	./tools/apply-lint.sh
@@ -163,7 +163,7 @@ build/test.data: $(CPYTHONLIB)
 		python $(FILEPACKAGER) test.data --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python$(PYMINOR)/test --js-output=test.js --export-name=globalThis.pyodide._module --exclude __pycache__ \
 	)
 	$(UGLIFYJS) build/test.js -o build/test.js
-  
+
 
 build/distutils.data: $(CPYTHONLIB)
 	( \

bin/pyodide

@@ -19,7 +19,7 @@
 TEST_PATH = ROOT_PATH / "src" / "tests"
 BUILD_PATH = ROOT_PATH / "build"
 
-sys.path.append(str(ROOT_PATH))
+sys.path.append(str(ROOT_PATH / "pyodide-build"))
 
 from pyodide_build.testing import set_webdriver_script_timeout, parse_driver_timeout
 

conftest.py

@@ -12,6 +12,7 @@
 base_dir = pathlib.Path(__file__).resolve().parent.parent
 path_dirs = [
     str(base_dir),
+    str(base_dir / "pyodide-build"),
     str(base_dir / "docs/sphinx_pyodide"),
     str(base_dir / "src/pyodide-py"),
     str(base_dir / "packages/micropip/micropip"),

docs/conf.py

@@ -30,7 +30,7 @@ load a package's dependencies automatically.
 If you wish to create a new package for Pyodide, the easiest place to start is
 with the {ref}`mkpkg tool <pyodide-mkpkg>`. If your package is on PyPI, just run:
 
-`bin/pyodide mkpkg $PACKAGE_NAME`
+`pyodide-build mkpkg $PACKAGE_NAME`
 
 This will generate a `meta.yaml` (see below) that should work out of the box
 for many pure Python packages. This tool will populate the latest version, download

docs/development/new-packages.md

@@ -20,12 +20,13 @@ and check that they are in your `PATH`.
 To run the pytest suite of tests, type on the command line:
 
 ```bash
-pytest src/ pyodide_build/ packages/*/test_*
+pytest src/ pyodide-build/ packages/*/test_*
 ```
 
 There are 3 test locations,
 - `src/tests/`: general Pyodide tests and tests running the CPython test suite
-- `pyodide_build/tests/`: tests related to Pyodide build system (do not require selenium to run)
+- `pyodide-build/pyodide_build/tests/`: tests related to Pyodide build system
+  (do not require selenium to run)
 - `packages/*/test_*`: package specific tests.
 
 ### Manual interactive testing
@@ -38,7 +39,7 @@ environment and the host system, run: `./run_docker`
 
 2. Now, this can be used to test the Pyodide builds running within the
 docker environment using external browser programs on the host system. To do
-this, run: `./bin/pyodide serve`
+this, run: `pyodide-build serve`
 
 3. This serves the ``build`` directory of the Pyodide project on port 8000.
     * To serve a different directory, use the ``--build_dir`` argument followed
@@ -77,7 +78,7 @@ make lint
 ### run_in_pyodide
 Many tests simply involve running a chunk of code in Pyodide and ensuring it
 doesn't error. In this case, one can use the `run_in_pyodide` decorate from
-`pyodide_build/testing.py`, e.g.
+`pyodide_build.testing`, e.g.
 
 ```python
 from pyodide_build.testing import run_in_pyodide

docs/development/testing.md

@@ -32,6 +32,12 @@ substitutions:
 - {{ Enhancement }} Added the {any}`PyProxy.clone` method.
   {pr}`1549`
 
+### pyodide-build
+
+- {{ Enhancement }} pyodide-build is now an installable Python package, with an identically named
+  CLI entrypoint that replaces `bin/pyodide` which is removed {pr}`1566`
+
+
 ## Version 0.17.0
 *April 21, 2020*
 

docs/project/changelog.md

@@ -1,10 +1,10 @@
 (pyodide-build-cli)=
-# pyodide_build CLI
+# pyodide-build CLI
 
 ```{eval-rst}
 .. sphinx_argparse_cli::
    :module: pyodide_build.__main__
    :func: make_parser
-   :prog: bin/pyodide
+   :prog: pyodide-build
    :title:
 ```

docs/usage/api/pyodide-build-cli.md

@@ -9,13 +9,14 @@ endif
 
 all:
 	mkdir -p $(PYODIDE_LIBRARIES)
-	PYTHONPATH=$(PYODIDE_LIBRARIES)/lib/python ../bin/pyodide buildall . ../build \
+	python3 -m pip install -e ../pyodide-build --no-deps --prefix $(PYODIDE_LIBRARIES)
+	PYTHONPATH="$(PYODIDE_LIBRARIES)/lib/python:$(PYODIDE_ROOT)/pyodide-build/" pyodide-build buildall . ../build \
 		--target=$(TARGETPYTHONROOT) $(ONLY_PACKAGES) --install-dir $(PYODIDE_LIBRARIES) --n-jobs $${PYODIDE_JOBS:-4}
 
 
 update-all:
 	for pkg in $$(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2); do \
-		../bin/pyodide mkpkg "$${pkg}" --update; \
+		pyodide-build mkpkg "$${pkg}" --update; \
 	done
 
 clean:

packages/Makefile

@@ -0,0 +1,10 @@
+# pyodide-build
+
+Tools for building Pyodide.
+
+See [https://github.com/pyodide/pyodide](https://github.com/pyodide/pyodide) for more information.
+
+## License
+
+Pyodide uses the [Mozilla Public License Version
+2.0](https://choosealicense.com/licenses/mpl-2.0/).

pyodide-build/README.md

@@ -55,7 +55,8 @@ def get_make_environment_vars():
     """Load environment variables from Makefile.envs
 
     This allows us to set all build vars in one place"""
-    __ROOTDIR = Path(__file__).parents[1].resolve()
+    # TODO: make this not rely on paths outside of pyodide-build
+    __ROOTDIR = Path(__file__).parents[2].resolve()
     environment = {}
     result = subprocess.run(
         ["make", "-f", str(__ROOTDIR / "Makefile.envs"), ".output_vars"],

pyodide-build/pyodide_build/__init__.py

@@ -40,7 +40,6 @@
 from pyodide_build._f2c_fixes import fix_f2c_clapack_calls
 
 
-TOOLSDIR = Path(common.get_make_flag("TOOLSDIR"))
 symlinks = set(["cc", "c++", "ld", "ar", "gcc", "gfortran"])
 
 
@@ -59,6 +58,7 @@ def collect_args(basename):
     It writes the arguments to the build.log, and then delegates to the real
     native compiler or linker.
     """
+    TOOLSDIR = Path(common.get_make_flag("TOOLSDIR"))
     # Remove the symlink compiler from the PATH, so we can delegate to the
     # native compiler
     env = dict(os.environ)
@@ -112,6 +112,7 @@ def make_symlinks(env):
     Makes sure all of the symlinks that make this script look like a compiler
     exist.
     """
+    TOOLSDIR = Path(common.get_make_flag("TOOLSDIR"))
     exec_path = Path(__file__).resolve()
     for symlink in symlinks:
         symlink_path = TOOLSDIR / symlink
@@ -130,6 +131,7 @@ def make_symlinks(env):
 
 
 def capture_compile(args):
+    TOOLSDIR = Path(common.get_make_flag("TOOLSDIR"))
     env = dict(os.environ)
     make_symlinks(env)
     env["PATH"] = str(TOOLSDIR) + ":" + os.environ["PATH"]

pyodide_build/__main__.py pyodide-build/pyodide_build/__main__.py

@@ -6,7 +6,7 @@
 from pyodide_build import buildall
 import pytest
 
-PACKAGES_DIR = (Path(__file__) / ".." / ".." / ".." / "packages").resolve()
+PACKAGES_DIR = (Path(__file__).parents[3] / "packages").resolve()
 
 
 def test_generate_dependency_graph():

pyodide_build/_f2c_fixes.py pyodide-build/pyodide_build/_f2c_fixes.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 import subprocess
 
-BASE_DIR = Path(__file__).parents[2]
+BASE_DIR = Path(__file__).parents[3]
 
 
 def test_run_docker_script():

pyodide_build/buildall.py pyodide-build/pyodide_build/buildall.py

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]

pyodide_build/buildpkg.py pyodide-build/pyodide_build/buildpkg.py

@@ -0,0 +1,30 @@
+[metadata]
+name = pyodide-build
+version = 0.18.0dev0
+author = Pyodide developers
+description = "Tools for building Pyodide"
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/pyodide/pyodide
+project_urls =
+    Bug Tracker = https://github.com/pyodide/pyodide/issues
+    Documentation = https://pyodide.org/en/stable/
+classifiers =
+    Programming Language :: Python :: 3
+    License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+    Operating System :: OS Independent
+
+[options]
+package_dir =
+    = .
+packages = find:
+python_requires = >=3.8
+install_requires =
+    pyyaml
+    cython<3.0
+[options.entry_points]
+console_scripts =
+    pyodide-build = pyodide_build.__main__:main
+
+[options.packages.find]
+where = .

pyodide_build/common.py pyodide-build/pyodide_build/common.py

@@ -0,0 +1,3 @@
+import setuptools
+
+setuptools.setup()

pyodide_build/io.py pyodide-build/pyodide_build/io.py

@@ -5,5 +5,5 @@ ROOT=`dirname ${BASH_SOURCE[0]}`
 # emsdk_env.sh is fairly noisy, and suppress error message if the file doesn't
 # exist yet (i.e. before building emsdk)
 source "$ROOT/emsdk/emsdk/emsdk_env.sh" 2> /dev/null || true
-export PATH=$ROOT/node_modules/.bin/:$PATH
+export PATH="$ROOT/node_modules/.bin/:$PATH:$ROOT/packages/.artifacts/bin/"
 export EM_DIR=$(dirname $(which emcc.py || echo "."))