diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 2b264924..2f5ef9d5 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -21,7 +21,7 @@ jobs: ~/.cache/sphinx ~/.ccache ~/.local - ~/.npm + ~/.bun key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }} - uses: actions/setup-python@v5 @@ -32,11 +32,14 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r requirements.docs.txt + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv --python 3.10 && source .venv/bin/activate + uv pip install --upgrade pip + uv pip install -r requirements.docs.txt - name: Build docs run: | + source .venv/bin/activate cd doc make html touch _build/html/.nojekyll diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55edf7b9..21b96fa4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -36,9 +36,7 @@ jobs: with: python-version: "3.10" - - uses: actions/setup-node@v4 - with: - node-version: 20 + - uses: oven-sh/setup-bun@v2 - name: Checkout uses: actions/checkout@v4 @@ -51,8 +49,8 @@ jobs: - uses: actions/cache@v4 with: path: | - ~/.npm - key: ${{ runner.os }}-npm-${{ hashFiles('baselayer_template_app/package.json') }} + ~/.bun + key: ${{ runner.os }}-bun-${{ hashFiles('baselayer_template_app/package.json') }} - uses: actions/cache@v4 with: @@ -66,8 +64,6 @@ jobs: - name: Install system dependencies run: | - cd baselayer - sudo apt update -y ### firefox installation @@ -75,7 +71,7 @@ jobs: sudo add-apt-repository ppa:mozillateam/ppa printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001' | sudo tee /etc/apt/preferences.d/mozilla-firefox - sudo apt install -y wget nodejs unzip firefox + sudo apt install -y wget unzip firefox # if nginx is already installed, remove it sudo apt remove -y nginx nginx-common nginx-core nginx-full @@ -86,31 +82,35 @@ jobs: sudo apt update -y sudo apt install nginx libnginx-mod-http-brotli-static libnginx-mod-http-brotli-filter -y + ### UV python package manager installation + curl -LsSf https://astral.sh/uv/install.sh | sh - pip install --upgrade pip - pip install wheel + cd baselayer_template_app - export NPM_PACKAGES="${HOME}/.npm-packages" - export PATH=${NPM_PACKAGES}/bin:$PATH - export NODE_PATH="$NPM_PACKAGES/lib/node_modules:$NODE_PATH" + uv venv --python 3.10 && source .venv/bin/activate - sudo npm -g install npm@latest + uv pip install --upgrade pip && uv pip install wheel + + bun upgrade which python; python --version - echo npm $(npm --version) - echo node $(node --version) + echo uv $(uv --version) + echo bun $(bun --version) nginx -v firefox --version - name: Install Geckodriver / Selenium run: | + cd baselayer_template_app + source .venv/bin/activate geckodriver --version - pip install selenium==4.8.3 + uv pip install selenium==4.8.3 python -c "import selenium; print(f'Selenium {selenium.__version__}')" - name: Write configuration & build DB run: | cd baselayer_template_app + source .venv/bin/activate cat << EOF > config.yaml database: @@ -144,6 +144,7 @@ jobs: - name: Test template app run: | cd baselayer_template_app + source .venv/bin/activate make test_headless - name: Upload logs diff --git a/.gitignore b/.gitignore index 76e4eda3..76532e63 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__/ node_modules/ *.pyc doc/_build +bun.lockb # These are built from templates conf/supervisor/supervisor.conf diff --git a/Makefile b/Makefile index 2d2ebed6..9f47b38b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL = /bin/bash -ESLINT=npx eslint +ESLINT=bun eslint .DEFAULT_GOAL := help @@ -10,6 +10,11 @@ FLAGS:=$(if $(FLAGS),$(FLAGS),--config=config.yaml) PYTHON=PYTHONPATH=. python ENV_SUMMARY=$(PYTHON) baselayer/tools/env_summary.py $(FLAGS) +# Set the Python package installer to use +# if uv is available set it to uv pip, otherwise use pip +# for that simply call the uv command, and if we get a non-zero exit code, we know it's not available +PYTHON_PGK_INSTALLER := $(shell command -v uv 2>&1 >/dev/null && echo "uv pip" || echo "pip") + # Flags are propagated to supervisord via the FLAGS environment variable # Inside of supervisord configuration files, you may reference them using # %(ENV_FLAGS)s @@ -25,7 +30,7 @@ B=\033[1m N=\033[0m bundle = static/build/main.bundle.js -rspack = npx rspack +rspack = bun rspack # NOTE: These targets are meant to be *included* in the parent app # Makefile. See end of this file for baselayer specific targets. @@ -41,7 +46,7 @@ help: @python ./baselayer/tools/makefile_to_help.py $(MAKEFILE_LIST) dependencies: README.md - @PYTHONPATH=. pip install packaging + @PYTHONPATH=. $(PYTHON_PGK_INSTALLER) install packaging setuptools wheel @baselayer/tools/check_app_environment.py @PYTHONPATH=. python baselayer/tools/pip_install_requirements.py baselayer/requirements.txt requirements.txt @./baselayer/tools/silent_monitor.py baselayer/tools/check_js_deps.sh @@ -146,10 +151,6 @@ test_report: ## Print report on failed tests test_report: @PYTHONPATH='.' baselayer/tools/junitxml_report.py test-results/junit.xml -# Call this target to see which Javascript dependencies are not up to date -check-js-updates: - ./baselayer/tools/check_js_updates.sh - # Lint targets lint-install: ## Install ESLint and a git pre-commit hook. lint-install: cp-lint-yaml lint-githook @@ -182,7 +183,7 @@ lint-githook: # Documentation targets, run from the `baselayer` directory baselayer_doc_reqs: - pip install -q -r requirements.docs.txt + $(PYTHON_PGK_INSTALLER) install -q -r requirements.docs.txt baselayer_html: | baselayer_doc_reqs export SPHINXOPTS=-W; make -C doc html diff --git a/doc/dev.md b/doc/dev.md index 2295d997..9e0ee8ef 100644 --- a/doc/dev.md +++ b/doc/dev.md @@ -129,10 +129,3 @@ We use ESLint to ensure that our JavaScript & JSX code is consistent and conform - Install ESLint using `make lint-install`. This will also install a git pre-commit hook so that any commit is linted before it is checked in. - Run `make lint` to perform a style check - -## Upgrading Javascript dependencies - -The `./tools/check_js_updates.sh` script uses -[`npm-check`](https://github.com/dylang/npm-check) to search updates -for packages defined in `package.json`. It then provides an -interactive interface for selecting new versions and performing the upgrade. diff --git a/doc/setup.md b/doc/setup.md index 7e9d2806..c576322c 100644 --- a/doc/setup.md +++ b/doc/setup.md @@ -38,7 +38,7 @@ See [below](#configuration) for more information on modifying the baselayer conf ### On Linux - Using `apt-get`: - `sudo apt-get install supervisor postgresql libpq-dev npm nodejs-legacy` + `sudo apt-get install supervisor postgresql libpq-dev nodejs-legacy` If you want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (better compression rates for the frontend), you have to install NGINX and the brotli module from another source with: @@ -51,6 +51,12 @@ See [below](#configuration) for more information on modifying the baselayer conf Otherwise, you can install NGINX normally with `sudo apt-get install nginx`. + Then, we install `Bun` (a javascript runtime, faster equivalent to `node+npm`): + + ``` + curl -fsSL https://bun.sh/install | bash + ``` + - It may be necessary to configure your database permissions: at the end of your `pg_hba.conf` (typically in `/etc/postgresql/13.3/main` or `/var/lib/pgsql/data`), add the following lines and restart PostgreSQL diff --git a/services/rspack/rspack.py b/services/rspack/rspack.py index 299e6a3f..40cc92ce 100644 --- a/services/rspack/rspack.py +++ b/services/rspack/rspack.py @@ -20,7 +20,7 @@ def run(cmd): if env.debug: log("Debug mode detected, launching rspack monitor") - p = run(["npx", "rspack", "--watch"]) + p = run(["bun", "rspack", "--watch"]) sys.exit(p.returncode) else: log("Production mode; not building JavaScript bundle") diff --git a/tools/check_app_environment.py b/tools/check_app_environment.py index 82771732..85a08d26 100755 --- a/tools/check_app_environment.py +++ b/tools/check_app_environment.py @@ -29,8 +29,7 @@ def output(cmd): lambda v: v.split("\n")[-1].split()[2], "12.0", ), - "npm": (["npm", "-v"], lambda v: v, "8.3.2"), - "node": (["node", "-v"], lambda v: v[1:], "16.14.0"), + "bun": (["bun", "-v"], lambda v: v, "1.1.33"), "python": (["python", "--version"], lambda v: v.split()[1], "3.8"), } diff --git a/tools/check_js_deps.sh b/tools/check_js_deps.sh index 0584dddb..26890f60 100755 --- a/tools/check_js_deps.sh +++ b/tools/check_js_deps.sh @@ -2,18 +2,28 @@ set -e -CHECKER="node_modules/.bin/check-dependencies" +# in this script, we want to use bun install to check if all the dependencies are installed +# if we run bun install and: +# - it contains and error message, print it out and exit +# - it contains "(no changes)" in the output, then we know all the dependencies are installed +# - otherwise, we tell the user that there are missing dependencies that were installed -if [[ ! -x ${CHECKER} ]]; then - npm install check-dependencies --legacy-peer-deps -fi +INSTALLER="bun install" + +# first run it and save the output +output=$(${INSTALLER} 2>&1) -# We suppress output for the next command because, annoyingly, it reports -# that a dependency is unsatisfied even if the --install flag is specified, -# and that package has been successfully installed -${CHECKER} --install +# if we got an error message, print it out and exit with an error +if [ $? -ne 0 ]; then + echo "✗ Error installing Javascript dependencies:" + echo "${output}" + exit 1 +fi -# Print report, if any unsatisfied dependencies remain -if ${CHECKER}; then - echo "✓ All Javascript dependencies satisfied." +# check if the output contains "(no changes)" +if echo "${output}" | grep -q "(no changes)"; then + echo "✓ All Javascript dependencies satisfied." +else + echo "✗ Some Javascript dependencies are unsatisfied." + echo "✓ Missing dependencies have been installed." fi diff --git a/tools/check_js_updates.sh b/tools/check_js_updates.sh deleted file mode 100755 index a9ec35ff..00000000 --- a/tools/check_js_updates.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -e - -CHECKER="npx npm-check" - -if ( ! $CHECKER --version > /dev/null 2>&1 ); then - echo "Update checker not found; installing." - npm install npm-check -fi - -${CHECKER} --skip-unused -u diff --git a/tools/pip_install_requirements.py b/tools/pip_install_requirements.py index 89c9debd..35c55ae7 100644 --- a/tools/pip_install_requirements.py +++ b/tools/pip_install_requirements.py @@ -5,6 +5,14 @@ from pkg_resources import DistributionNotFound, Requirement, VersionConflict from status import status +uv = True +# check if the uv command is available +# if not, use pip directly +try: + subprocess.run(["uv", "--version"], stdout=subprocess.PIPE) +except FileNotFoundError: + uv = False + if len(sys.argv) < 2: print( "Usage: pip_install_requirements.py requirements.txt [requirements_other.txt]" @@ -19,9 +27,14 @@ def pip(req_files): - args = ["pip", "install"] + if uv: + args = ["uv", "pip", "install"] + else: + args = ["pip", "install"] + for req_file in req_files: args.extend(["-r", req_file]) + p = subprocess.Popen( args, stdout=subprocess.PIPE, diff --git a/tools/update_eslint.sh b/tools/update_eslint.sh deleted file mode 100755 index cbd6ac8d..00000000 --- a/tools/update_eslint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -export PKG=eslint-config-airbnb -npm info "${PKG}@latest" peerDependencies --json | \ - command sed 's/[\{\},]//g ; s/: /@/g' | \ - xargs npm install --save-dev "${PKG}@latest"