From 24c7d1e7cc379c2b5a05fb2ac08608de5e30b7fd Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Tue, 29 Apr 2025 17:42:29 +0200 Subject: [PATCH 01/17] Don't upgrade pip, setuptools, wheels --- .github/workflows/main_unit_tests.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index 3804edbd..360b1464 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -42,11 +42,15 @@ jobs: submodules: 'recursive' path: pace/${{inputs.component_name}} + - name: Print versions + run: | + python --version + pip --version + - name: install packages if: ${{ ! inputs.component_trigger }} run: | cd ${GITHUB_WORKSPACE}/pace - pip3 install --upgrade pip setuptools wheel pip3 install -r requirements_dev.txt -c constraints.txt - name: "External trigger: Install packages" @@ -55,7 +59,6 @@ jobs: # to avoid false positives in package dependency resolution. run: | cd ${GITHUB_WORKSPACE}/pace - pip3 install --upgrade pip setuptools wheel pip3 install -r requirements_dev.txt - name: prepare input files From 01a82dadc973e21cc72b353686c017776410dd4f Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 10:58:09 +0200 Subject: [PATCH 02/17] WIP: dependencies handling --- .github/workflows/main_unit_tests.yaml | 17 +- README.md | 13 +- constraints.txt | 537 ------------------------- examples/generate_eta_files.py | 19 +- pyproject.toml | 2 +- requirements_dev.txt | 20 +- requirements_docs.txt | 1 + requirements_lint.txt | 1 - setup.py | 42 +- 9 files changed, 58 insertions(+), 594 deletions(-) delete mode 100644 constraints.txt delete mode 100644 requirements_lint.txt diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index 360b1464..1a3ee88b 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -46,32 +46,23 @@ jobs: run: | python --version pip --version + pip list - name: install packages - if: ${{ ! inputs.component_trigger }} run: | cd ${GITHUB_WORKSPACE}/pace - pip3 install -r requirements_dev.txt -c constraints.txt - - - name: "External trigger: Install packages" - if: ${{ inputs.component_trigger }} - # Ignore `constraints.txt` when running as part of the NDSL pipeline - # to avoid false positives in package dependency resolution. - run: | - cd ${GITHUB_WORKSPACE}/pace - pip3 install -r requirements_dev.txt + pip install .[prod,test] - name: prepare input files run: | cd ${GITHUB_WORKSPACE}/pace mkdir tests/main/input - python3 examples/generate_eta_files.py - mv *eta*.nc tests/main/input + python examples/generate_eta_files.py tests/main/input - name: run tests run: | cd ${GITHUB_WORKSPACE}/pace - pytest -x tests/main + python -m pytest -x tests/main - name: run baroclinic run: | diff --git a/README.md b/README.md index 58d674e6..d91ff7ff 100644 --- a/README.md +++ b/README.md @@ -60,14 +60,14 @@ git submodule update --init --recursive We recommend creating a python `venv` or `conda` environment specifically for Pace. ```shell -python3 -m venv venv_name -source venv_name/bin/activate +python -m venv .venv +source .venv/bin/activate ``` -Inside of your pace `venv` or conda environment pip install the Python requirements, GT4Py, and Pace: +Inside of your pace `venv` or `conda` environment, pip install the Python requirements, GT4Py, and Pace: ```shell -pip3 install -r requirements_dev.txt -c constraints.txt +pip install -r requirements_dev.txt ``` Shell scripts to install Pace on specific machines such as Gaea can be found in `examples/build_scripts/`. @@ -80,8 +80,7 @@ Before starting any run, including unit tests, the user must ensure that the pro ```shell mkdir tests/main/input -python3 examples/generate_eta_files.py -mv *eta*.nc tests/main/input +python examples/generate_eta_files.py tests/main/input ``` These commands will generate the files necessary and place them in the `tests/main/input` directory. Once the files are generated the `baroclinic_c12.yaml` configuration can be used to generate a run: @@ -93,7 +92,7 @@ mpirun -n 6 python3 -m pace.run examples/configs/baroclinic_c12.yaml mpirun -n 6 --oversubscribe python3 -m pace.run examples/configs/baroclinic_c12.yaml ``` -After the run completes, you will see an output direcotry `output.zarr`. An example to visualize the output is provided in `examples/plot_output.py`. See the [driver example](examples/README.md) section for more details. +After the run completes, you will see an output directory `output.zarr`. An example to visualize the output is provided in `examples/plot_output.py`. See the [driver example](examples/README.md) section for more details. ### Environment variable configuration diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index 44dee2aa..00000000 --- a/constraints.txt +++ /dev/null @@ -1,537 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --output-file=constraints.txt requirements_dev.txt requirements_docs.txt requirements_lint.txt -# -aenum==3.1.15 - # via dace -aiohttp==3.9.5 - # via gcsfs -aiosignal==1.3.1 - # via aiohttp -alabaster==0.7.12 - # via sphinx -appdirs==1.4.4 - # via fv3config -asciitree==0.3.3 - # via zarr -asttokens==2.4.1 - # via - # devtools - # stack-data -astunparse==1.6.3 - # via dace -attrs==23.2.0 - # via - # aiohttp - # gt4py - # jsonschema - # pytest-subtests - # referencing -babel==2.9.1 - # via sphinx -backports-entry-points-selectable==1.1.1 - # via virtualenv -black==24.4.0 - # via gt4py -boltons==24.0.0 - # via gt4py -cached-property==1.5.2 - # via gt4py -cachetools==5.3.3 - # via google-auth -certifi==2024.2.2 - # via - # netcdf4 - # requests -cfgv==3.3.1 - # via pre-commit -cftime==1.6.3 - # via - # -r requirements_dev.txt - # ndsl - # netcdf4 -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # black - # dask - # gt4py -cloudpickle==3.0.0 - # via dask -cmake==3.29.2 - # via - # dace - # gt4py -comm==0.2.2 - # via ipykernel -commonmark==0.9.1 - # via recommonmark -contourpy==1.3.0 - # via matplotlib -coverage==7.5.0 - # via pytest-cov -cycler==0.12.1 - # via matplotlib -cytoolz==0.12.3 - # via gt4py -dacite==1.8.1 - # via - # fv3config - # pace -dask==2024.4.2 - # via - # -r requirements_dev.txt - # ndsl -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via - # gcsfs - # ipython -deepdiff==7.0.1 - # via gt4py -devtools==0.12.2 - # via gt4py -dill==0.3.8 - # via dace -distlib==0.3.2 - # via virtualenv -distro==1.9.0 - # via scikit-build -docutils==0.16 - # via - # recommonmark - # sphinx - # sphinx-rtd-theme -executing==2.0.1 - # via - # devtools - # stack-data -f90nml==1.4.4 - # via - # -r requirements_dev.txt - # fv3config - # ndsl - # pyfv3 - # pyshield -factory-boy==3.3.0 - # via gt4py -faker==24.11.0 - # via factory-boy -fasteners==0.19 - # via zarr -fastjsonschema==2.19.1 - # via nbformat -filelock==3.0.12 - # via virtualenv -fonttools==4.53.1 - # via matplotlib -fparser==0.1.4 - # via dace -frozendict==2.4.2 - # via gt4py -frozenlist==1.4.1 - # via - # aiohttp - # aiosignal -fsspec==2024.3.1 - # via - # dask - # fv3config - # gcsfs - # ndsl -fv3config==0.9.0 - # via -r requirements_dev.txt -gcsfs==2024.3.1 - # via fv3config -google-api-core==2.18.0 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.29.0 - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage -google-auth-oauthlib==1.2.0 - # via gcsfs -google-cloud-core==2.4.1 - # via google-cloud-storage -google-cloud-storage==2.16.0 - # via gcsfs -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via google-cloud-storage -googleapis-common-protos==1.63.0 - # via google-api-core -gprof2dot==2022.7.29 - # via pytest-profiling -gridtools-cpp==2.3.4 - # via gt4py -h5netcdf==1.3.0 - # via ndsl -h5py==3.11.0 - # via h5netcdf -identify==2.2.13 - # via pre-commit -idna==3.7 - # via - # requests - # yarl -imagesize==1.2.0 - # via sphinx -importlib-metadata==7.1.0 - # via dask -iniconfig==2.0.0 - # via pytest -ipykernel==6.29.4 - # via nbmake -ipython==8.23.0 - # via ipykernel -jedi==0.19.1 - # via ipython -jinja2==3.1.3 - # via - # dace - # gt4py - # sphinx -jsonschema==4.21.1 - # via nbformat -jsonschema-specifications==2023.12.1 - # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbformat -kiwisolver==1.4.7 - # via matplotlib -lark==1.1.9 - # via gt4py -locket==1.0.0 - # via partd -mako==1.3.3 - # via gt4py -markupsafe==2.1.5 - # via - # jinja2 - # mako -matplotlib==3.9.2 - # via ndsl -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mpi4py==3.1.5 - # via - # -r requirements_dev.txt - # ndsl - # pace -mpmath==1.3.0 - # via sympy -multidict==6.0.5 - # via - # aiohttp - # yarl -mypy-extensions==1.0.0 - # via black -nanobind==1.9.2 - # via gt4py -nbclient==0.6.8 - # via nbmake -nbformat==5.10.4 - # via - # nbclient - # nbmake -nbmake==1.5.3 - # via -r requirements_dev.txt -nest-asyncio==1.6.0 - # via - # ipykernel - # nbclient -netcdf4==1.7.1 - # via - # -r requirements_dev.txt - # ndsl - # pace -networkx==3.3 - # via dace -ninja==1.11.1.1 - # via gt4py -nodeenv==1.6.0 - # via pre-commit -numcodecs==0.12.1 - # via zarr -numpy==1.26.4 - # via - # -r requirements_dev.txt - # cftime - # contourpy - # dace - # gt4py - # h5py - # matplotlib - # ndsl - # netcdf4 - # numcodecs - # pace - # pandas - # pyfv3 - # pyshield - # scipy - # xarray - # zarr -oauthlib==3.2.2 - # via requests-oauthlib -ordered-set==4.1.0 - # via deepdiff -packaging==24.0 - # via - # black - # dace - # dask - # gt4py - # h5netcdf - # ipykernel - # matplotlib - # pytest - # scikit-build - # setuptools-scm - # sphinx - # xarray -pandas==2.2.2 - # via xarray -parso==0.8.4 - # via jedi -partd==1.4.1 - # via dask -pathspec==0.12.1 - # via black -pexpect==4.9.0 - # via ipython -pillow==10.4.0 - # via matplotlib -platformdirs==2.6.2 - # via - # black - # jupyter-core - # virtualenv -pluggy==1.5.0 - # via pytest -ply==3.11 - # via dace -pre-commit==2.14.0 - # via -r requirements_lint.txt -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via google-api-core -protobuf==4.25.3 - # via - # google-api-core - # googleapis-common-protos - # proto-plus -psutil==5.9.8 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pybind11==2.12.0 - # via gt4py -pygments==2.17.2 - # via - # devtools - # ipython - # nbmake - # sphinx -pyparsing==3.1.4 - # via matplotlib -pytest==8.1.1 - # via - # -r requirements_dev.txt - # nbmake - # pytest-cov - # pytest-datadir - # pytest-profiling - # pytest-regressions - # pytest-subtests -pytest-cov==5.0.0 - # via -r requirements_dev.txt -pytest-datadir==1.5.0 - # via pytest-regressions -pytest-profiling==1.7.0 - # via -r requirements_dev.txt -pytest-regressions==2.5.0 - # via -r requirements_dev.txt -pytest-subtests==0.12.1 - # via -r requirements_dev.txt -python-dateutil==2.9.0.post0 - # via - # faker - # jupyter-client - # matplotlib - # pandas -pytz==2024.1 - # via - # babel - # pandas -pyyaml==6.0.1 - # via - # dace - # dask - # fv3config - # pace - # pre-commit - # pytest-regressions -pyzmq==26.0.2 - # via - # ipykernel - # jupyter-client -recommonmark==0.7.1 - # via -r requirements_docs.txt -referencing==0.35.0 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via - # gcsfs - # google-api-core - # google-cloud-storage - # requests-oauthlib - # sphinx -requests-oauthlib==2.0.0 - # via google-auth-oauthlib -rpds-py==0.18.0 - # via - # jsonschema - # referencing -rsa==4.9 - # via google-auth -scikit-build==0.18.1 - # via dace -scipy==1.13.0 - # via - # -r requirements_dev.txt - # ndsl -setuptools-scm==8.0.4 - # via fparser -six==1.16.0 - # via - # asttokens - # astunparse - # pytest-profiling - # python-dateutil - # virtualenv -snowballstemmer==2.1.0 - # via sphinx -sphinx==4.1.2 - # via - # -r requirements_docs.txt - # recommonmark - # sphinx-argparse - # sphinx-gallery - # sphinx-rtd-theme -sphinx-argparse==0.3.1 - # via -r requirements_docs.txt -sphinx-gallery==0.10.1 - # via -r requirements_docs.txt -sphinx-rtd-theme==0.5.2 - # via -r requirements_docs.txt -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -stack-data==0.6.3 - # via ipython -sympy==1.12 - # via dace -tabulate==0.9.0 - # via gt4py -toml==0.10.2 - # via pre-commit -toolz==0.12.1 - # via - # cytoolz - # dask - # partd -tornado==6.4 - # via - # ipykernel - # jupyter-client -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbformat -typing-extensions==4.12.2 - # via - # gt4py - # ipython - # setuptools-scm -tzdata==2024.1 - # via pandas -urllib3==2.2.1 - # via requests -virtualenv==20.7.2 - # via pre-commit -wcwidth==0.2.13 - # via prompt-toolkit -websockets==12.0 - # via dace -wheel==0.43.0 - # via - # astunparse - # scikit-build -xarray==2025.01.2 - # via - # -r requirements_dev.txt - # ndsl - # pace - # pyfv3 - # pyshield -xxhash==3.0.0 - # via gt4py -yarl==1.9.4 - # via aiohttp -zarr==2.18.2 - # via - # -r requirements_dev.txt - # pace -zipp==3.18.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/generate_eta_files.py b/examples/generate_eta_files.py index 1fb4d5ee..4a13ba10 100755 --- a/examples/generate_eta_files.py +++ b/examples/generate_eta_files.py @@ -1,16 +1,25 @@ +import sys import numpy as np import xarray as xr +from pathlib import Path """ This notebook uses the python xarray module to create an eta_file containing ak and bk coefficients for km=79 and km=91. The coefficients are written out to -eta79.nc and eta91.nc netcdf files respectively +`eta79.nc` and `eta91.nc` netcdf files respectively in the +given folder (defaults to `./` if not provided). -To run this script: `python3 ./generate_eta_files.py` +To run this script: `python3 ./generate_eta_files.py ` """ +folder = Path("." if len(sys.argv) < 2 else sys.argv[1]) +if not folder.exists(): + raise ValueError(f"Cannot open folder '{folder}'. Make sure it exists.") +if not folder.is_dir(): + raise ValueError(f"Expected '{folder}' to be a directory.") + # km = 79 ak = xr.DataArray( dims=["km1"], @@ -189,7 +198,7 @@ ), ) coefficients = xr.Dataset(data_vars={"ak": ak, "bk": bk}) -coefficients.to_netcdf("eta79.nc") +coefficients.to_netcdf(Path(folder) / "eta79.nc") # km = 91 @@ -394,6 +403,4 @@ ), ) coefficients = xr.Dataset(data_vars={"ak": ak, "bk": bk}) -coefficients.to_netcdf("eta91.nc") - -# km = +coefficients.to_netcdf(Path(folder) / "eta91.nc") diff --git a/pyproject.toml b/pyproject.toml index b76d062e..8737b121 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [tool.black] line-length = 88 -target_version = ['py36', 'py37', 'py38'] +target_version = ['py38', 'py39', 'py310', 'py311'] diff --git a/requirements_dev.txt b/requirements_dev.txt index 89271475..1a8610e2 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,20 +1,4 @@ -pytest -pytest-subtests -pytest-regressions -pytest-profiling -pytest-cov -scipy -nbmake -mpi4py -xarray -zarr <3.0.0 -dask -netCDF4 -cftime -fv3config -f90nml -numpy -e NDSL --e pySHiELD -e pyFV3 --e . +-e pySHiELD +-e .[lint,test] diff --git a/requirements_docs.txt b/requirements_docs.txt index 54bae3b4..a36d014b 100644 --- a/requirements_docs.txt +++ b/requirements_docs.txt @@ -1,3 +1,4 @@ +fv3config recommonmark sphinx sphinx-argparse diff --git a/requirements_lint.txt b/requirements_lint.txt deleted file mode 100644 index 416634f5..00000000 --- a/requirements_lint.txt +++ /dev/null @@ -1 +0,0 @@ -pre-commit diff --git a/setup.py b/setup.py index a095679b..6f9fbb4e 100644 --- a/setup.py +++ b/setup.py @@ -1,32 +1,49 @@ import os from pathlib import Path -from typing import List - from setuptools import find_namespace_packages, setup +from typing import List def local_pkg(name: str, relative_path: str) -> str: """Returns an absolute path to a local package.""" - path = f"{name} @ file://{Path(os.path.abspath(__file__)).parent / relative_path}" - return path + return f"{name} @ file://{Path(os.path.abspath(__file__)).parent / relative_path} " requirements: List[str] = [ - "ndsl", "dacite", - "pyyaml", - "mpi4py", - "numpy", - "netCDF4", + "f90nml", + "numpy < 2.0.0", # numpy 2.x has breaking API changes "xarray", - "zarr", + "zarr < 3.0.0", # zarr 3.x has breaking API changes +] + +prod_requirements = [ + local_pkg("NDSL", "NDSL"), + local_pkg("pyFV3", "pyFV3"), + local_pkg("pySHiELD", "pySHiELD"), ] +test_requirements = [ + "mpi4py", + "nbmake", + "pytest", +] + +lint_requirements = [ + "pre-commit" +] + +extras_require = { + "lint": lint_requirements, + "prod": prod_requirements, + "test": test_requirements, +} + setup( author="Allen Institute for AI", author_email="oliver.elbert@noaa.gov", - python_requires=">=3.8", + python_requires=">=3.8,<3.12", classifiers=[ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", @@ -35,8 +52,11 @@ def local_pkg(name: str, relative_path: str) -> str: "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], install_requires=requirements, + extras_require=extras_require, name="pace", license="BSD license", packages=find_namespace_packages(include=["pace", "pace.*"]), From 01aab90667493ba7f91c8b61a426492b7df950df Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 11:29:51 +0200 Subject: [PATCH 03/17] fix linting workflow --- .github/workflows/lint.yaml | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 0d15d592..48ea1b9b 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,21 +6,19 @@ jobs: lint: runs-on: ubuntu-latest steps: - - name: Checkout Pace repository - uses: actions/checkout@v3.5.2 - with: - submodules: 'recursive' - - name: Step Python 3.11.7 - uses: actions/setup-python@v4.6.0 - with: - python-version: '3.11.7' - - name: Install OpenMPI for gt4py - run: | - sudo apt-get install libopenmpi-dev - - name: Install Python packages - run: | - python -m pip install --upgrade pip - pip install -r requirements_dev.txt -r requirements_lint.txt - - name: Run lint via pre-commit - run: | - pre-commit run --all-files + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Step Python 3.11.9 + uses: actions/setup-python@v5 + with: + python-version: '3.11.9' + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run lint via pre-commit + run: | + pre-commit run --all-files From 361fa704beac97f7c3687c989fd44ec20b19276c Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 11:36:36 +0200 Subject: [PATCH 04/17] fix linting even more --- .github/workflows/main_unit_tests.yaml | 8 ++++---- examples/generate_eta_files.py | 3 ++- pyproject.toml | 2 +- setup.py | 11 +++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index 1a3ee88b..a7ffbf53 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -48,23 +48,23 @@ jobs: pip --version pip list - - name: install packages + - name: Install packages run: | cd ${GITHUB_WORKSPACE}/pace pip install .[prod,test] - - name: prepare input files + - name: Prepare input files run: | cd ${GITHUB_WORKSPACE}/pace mkdir tests/main/input python examples/generate_eta_files.py tests/main/input - - name: run tests + - name: Run tests run: | cd ${GITHUB_WORKSPACE}/pace python -m pytest -x tests/main - - name: run baroclinic + - name: Run baroclinic test case run: | cd ${GITHUB_WORKSPACE}/pace mpiexec -np 6 --oversubscribe python -m pace.run examples/configs/baroclinic_c12.yaml diff --git a/examples/generate_eta_files.py b/examples/generate_eta_files.py index 4a13ba10..78f7cf34 100755 --- a/examples/generate_eta_files.py +++ b/examples/generate_eta_files.py @@ -1,8 +1,9 @@ import sys +from pathlib import Path + import numpy as np import xarray as xr -from pathlib import Path """ This notebook uses the python xarray module diff --git a/pyproject.toml b/pyproject.toml index 8737b121..0ccea7b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [tool.black] line-length = 88 -target_version = ['py38', 'py39', 'py310', 'py311'] +target_version = ['py38'] diff --git a/setup.py b/setup.py index 6f9fbb4e..d904dada 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ import os from pathlib import Path -from setuptools import find_namespace_packages, setup from typing import List +from setuptools import find_namespace_packages, setup + def local_pkg(name: str, relative_path: str) -> str: """Returns an absolute path to a local package.""" @@ -12,9 +13,9 @@ def local_pkg(name: str, relative_path: str) -> str: requirements: List[str] = [ "dacite", "f90nml", - "numpy < 2.0.0", # numpy 2.x has breaking API changes + "numpy < 2.0.0", # numpy 2.x has breaking API changes "xarray", - "zarr < 3.0.0", # zarr 3.x has breaking API changes + "zarr < 3.0.0", # zarr 3.x has breaking API changes ] prod_requirements = [ @@ -29,9 +30,7 @@ def local_pkg(name: str, relative_path: str) -> str: "pytest", ] -lint_requirements = [ - "pre-commit" -] +lint_requirements = ["pre-commit"] extras_require = { "lint": lint_requirements, From ffe411c4fa5a674a1259a1446c4df44c038f2ece Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 14:16:00 +0200 Subject: [PATCH 05/17] remove editable installs of NDSL/pyFV3/pySHiELD in dev setup --- .github/workflows/main_unit_tests.yaml | 2 +- requirements_dev.txt | 3 --- setup.py | 10 +++------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index a7ffbf53..70c23f89 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -51,7 +51,7 @@ jobs: - name: Install packages run: | cd ${GITHUB_WORKSPACE}/pace - pip install .[prod,test] + pip install .[test] - name: Prepare input files run: | diff --git a/requirements_dev.txt b/requirements_dev.txt index 1a8610e2..0d6997d0 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1 @@ --e NDSL --e pyFV3 --e pySHiELD -e .[lint,test] diff --git a/setup.py b/setup.py index d904dada..ab7d927d 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,9 @@ def local_pkg(name: str, relative_path: str) -> str: requirements: List[str] = [ + local_pkg("NDSL", "NDSL"), + local_pkg("pyFV3", "pyFV3"), + local_pkg("pySHiELD", "pySHiELD"), "dacite", "f90nml", "numpy < 2.0.0", # numpy 2.x has breaking API changes @@ -18,12 +21,6 @@ def local_pkg(name: str, relative_path: str) -> str: "zarr < 3.0.0", # zarr 3.x has breaking API changes ] -prod_requirements = [ - local_pkg("NDSL", "NDSL"), - local_pkg("pyFV3", "pyFV3"), - local_pkg("pySHiELD", "pySHiELD"), -] - test_requirements = [ "mpi4py", "nbmake", @@ -34,7 +31,6 @@ def local_pkg(name: str, relative_path: str) -> str: extras_require = { "lint": lint_requirements, - "prod": prod_requirements, "test": test_requirements, } From e5741e101ea810c2d7c2bd70d88d700e0fba47d0 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 14:59:29 +0200 Subject: [PATCH 06/17] Use pip install -e .[test,lint] directly instead of requirements_dev.txt --- README.md | 4 ++-- examples/build_scripts/build_gaea_c4.sh | 5 ++--- examples/build_scripts/build_gaea_c5.sh | 5 ++--- examples/build_scripts/build_gaea_c5_gcc.sh | 5 ++--- examples/build_scripts/build_ppan.sh | 7 +++---- examples/build_scripts/build_ppan_intel.sh | 7 +++---- examples/create_venv.sh | 5 ++--- requirements_dev.txt | 1 - 8 files changed, 16 insertions(+), 23 deletions(-) delete mode 100644 requirements_dev.txt diff --git a/README.md b/README.md index d91ff7ff..1229520b 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,10 @@ python -m venv .venv source .venv/bin/activate ``` -Inside of your pace `venv` or `conda` environment, pip install the Python requirements, GT4Py, and Pace: +Inside of your pace `venv` or `conda` environment, pip install pace (as editable install) with the `lint` and `test` extras: ```shell -pip install -r requirements_dev.txt +pip install -e .[lint,test] ``` Shell scripts to install Pace on specific machines such as Gaea can be found in `examples/build_scripts/`. diff --git a/examples/build_scripts/build_gaea_c4.sh b/examples/build_scripts/build_gaea_c4.sh index def7af3f..07d7229b 100644 --- a/examples/build_scripts/build_gaea_c4.sh +++ b/examples/build_scripts/build_gaea_c4.sh @@ -19,9 +19,8 @@ cd pace # create a conda environment for pace conda create -y --name my_name python=3.8 -# enter the environment and update it +# enter the environment conda activate my_name -pip3 install --upgrade pip setuptools wheel # install the Pace dependencies, GT4Py, and Pace -pip3 install -r requirements_dev.txt -c constraints.txt +pip3 install . diff --git a/examples/build_scripts/build_gaea_c5.sh b/examples/build_scripts/build_gaea_c5.sh index 94ad5611..07dcbcd2 100644 --- a/examples/build_scripts/build_gaea_c5.sh +++ b/examples/build_scripts/build_gaea_c5.sh @@ -19,9 +19,8 @@ cd pace # create a conda environment for pace conda create -y --name my_name python=3.8 -# enter the environment and update it +# enter the environment conda activate my_name -pip3 install --upgrade pip setuptools wheel # install the Pace dependencies, GT4Py, and Pace -pip3 install -r requirements_dev.txt -c constraints.txt +pip3 install . diff --git a/examples/build_scripts/build_gaea_c5_gcc.sh b/examples/build_scripts/build_gaea_c5_gcc.sh index 87bac83a..26fc42a9 100644 --- a/examples/build_scripts/build_gaea_c5_gcc.sh +++ b/examples/build_scripts/build_gaea_c5_gcc.sh @@ -28,9 +28,8 @@ cd pace # create a conda environment for pace conda create -y --name my_name python=3.11.7 -# enter the environment and update it +# enter the environment conda activate my_name -pip3 install --upgrade pip setuptools wheel # install the Pace dependencies, GT4Py, and Pace -pip3 install -r requirements_dev.txt -c constraints.txt +pip3 install . diff --git a/examples/build_scripts/build_ppan.sh b/examples/build_scripts/build_ppan.sh index d6bf1bbd..3927d02f 100644 --- a/examples/build_scripts/build_ppan.sh +++ b/examples/build_scripts/build_ppan.sh @@ -24,12 +24,11 @@ cd $PACE_DIR # create a conda environment with cartopy and its dependencies installed conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.8 matplotlib==3.5.2 cartopy==0.18.0 -# enter the environment and update it +# enter the environment conda activate $ENVIRONMENT_NAME -pip3 install --upgrade --no-cache-dir pip setuptools wheel -# install the Pace dependencies, GT4Py, and Pace -pip3 install --no-cache-dir -r requirements_dev.txt -c constraints.txt +# install Pace dependencies and its dependencies +pip3 install --no-cache-dir . # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/build_scripts/build_ppan_intel.sh b/examples/build_scripts/build_ppan_intel.sh index c14dad78..c3a78bc8 100644 --- a/examples/build_scripts/build_ppan_intel.sh +++ b/examples/build_scripts/build_ppan_intel.sh @@ -31,12 +31,11 @@ cd $PACE_DIR # create a conda environment with cartopy and its dependencies installed conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.11.7 matplotlib==3.5.2 cartopy==0.18.0 -# enter the environment and update it +# enter the environment conda activate $ENVIRONMENT_NAME -pip3 install --upgrade --no-cache-dir pip setuptools wheel -# install the Pace dependencies, GT4Py, and Pace -pip3 install --no-cache-dir -r requirements_dev.txt -c constraints.txt +# install Pace dependencies and its dependencies +pip3 install --no-cache-dir . # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/create_venv.sh b/examples/create_venv.sh index 312932f1..58e26119 100755 --- a/examples/create_venv.sh +++ b/examples/create_venv.sh @@ -10,10 +10,9 @@ python3 -m venv venv . venv/bin/activate rundir=$(pwd) -cd ${SCRIPT_DIR}/../../ +cd ${SCRIPT_DIR}/../ -pip3 install --upgrade setuptools wheel -pip3 install -r requirements_dev.txt -c constraints.txt +pip3 install -e .[test,lint] deactivate cd $rundir diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 0d6997d0..00000000 --- a/requirements_dev.txt +++ /dev/null @@ -1 +0,0 @@ --e .[lint,test] From b3ebb94b8abbf77d9750b221aeecca1fcb9b2cb9 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:03:01 +0200 Subject: [PATCH 07/17] forgot to push some changes --- README.md | 2 +- examples/README.md | 2 ++ examples/build_scripts/build_gaea_c4.sh | 2 +- examples/build_scripts/build_gaea_c5_gcc.sh | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1229520b..1159b794 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Pace is an implementation of the FV3GFS / SHiELD atmospheric model developed by 🚧 **WARNING** This repo is under active development - supported features and procedures can change rapidly and without notice. 🚧 -The repository model code is split between [pyFV3](https://github.com/NOAA-GFDL/pyFV3) for the dynamical core and [pySHiELD](https://github.com/NOAA-GFDL/pySHiELD) for the physics parametrization. A full depencies looks like the following: +The repository model code is split between [pyFV3](https://github.com/NOAA-GFDL/pyFV3) for the dynamical core and [pySHiELD](https://github.com/NOAA-GFDL/pySHiELD) for the physics parametrization. A full dependencies looks like the following: ```mermaid flowchart TD diff --git a/examples/README.md b/examples/README.md index 55143967..873e04aa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -89,12 +89,14 @@ Another example is `baroclinic_init.py`, which initializes a barcolinic wave and ```bash $ mpirun -n 6 python3 baroclinic_init.py ./configs/baroclinic_c12.yaml ``` + ## Docker To run a baroclinic c12 case with Docker in a single command, run `run_docker.sh`. This example will start from the Python 3.8 docker image, install extra dependencies and Python packages, and execute the example, leaving the output in this directory. To visualize the output, two example scripts are provided: + 1. `plot_output.py`: To use it, you must install matplotlib (e.g. with `pip install matplotlib`). 2. `plot_cube.py`: this uses plotting tools in [fv3viz](https://github.com/ai2cm/fv3net/tree/master/external/fv3viz). Note the requirements aren't part of pace by default and need to be installed accordingly. It is recommended to use the post processing docker provided at the top level `docker/postprocessing.Dockerfile`. diff --git a/examples/build_scripts/build_gaea_c4.sh b/examples/build_scripts/build_gaea_c4.sh index 07d7229b..bffaed7a 100644 --- a/examples/build_scripts/build_gaea_c4.sh +++ b/examples/build_scripts/build_gaea_c4.sh @@ -22,5 +22,5 @@ conda create -y --name my_name python=3.8 # enter the environment conda activate my_name -# install the Pace dependencies, GT4Py, and Pace +# install Pace dependencies and its dependencies pip3 install . diff --git a/examples/build_scripts/build_gaea_c5_gcc.sh b/examples/build_scripts/build_gaea_c5_gcc.sh index 26fc42a9..61231ff2 100644 --- a/examples/build_scripts/build_gaea_c5_gcc.sh +++ b/examples/build_scripts/build_gaea_c5_gcc.sh @@ -31,5 +31,5 @@ conda create -y --name my_name python=3.11.7 # enter the environment conda activate my_name -# install the Pace dependencies, GT4Py, and Pace +# install Pace dependencies and its dependencies pip3 install . From 690eae7fb01065f5b49d6b1fe95c2e0581258d79 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:38:20 +0200 Subject: [PATCH 08/17] re-adding constraints.txt file --- .github/workflows/main_unit_tests.yaml | 11 +- CONTRIBUTING.md | 11 +- README.md | 24 +- constraints.txt | 230 ++++++++++++++++++++ examples/build_scripts/build_gaea_c4.sh | 2 +- examples/build_scripts/build_gaea_c5.sh | 2 +- examples/build_scripts/build_gaea_c5_gcc.sh | 2 +- examples/build_scripts/build_ppan.sh | 2 +- examples/build_scripts/build_ppan_intel.sh | 2 +- examples/create_venv.sh | 2 +- setup.py | 8 +- 11 files changed, 279 insertions(+), 17 deletions(-) create mode 100644 constraints.txt diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index 70c23f89..d7a1cb90 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -49,9 +49,18 @@ jobs: pip list - name: Install packages + if: ${{ ! inputs.component_trigger }} run: | cd ${GITHUB_WORKSPACE}/pace - pip install .[test] + pip install -c constraints.txt .[test] + + - name: "External trigger: Install packages" + if: ${{ inputs.component_trigger }} + # Ignore `constraints.txt` when running as part of the NDSL pipeline + # to avoid false positives in package dependency resolution. + run: | + cd ${GITHUB_WORKSPACE}/pace + pip3 install .[test] - name: Prepare input files run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5671f561..3205f125 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,23 +1,22 @@ # Contributing -pace is actively developed by AI2, so please contact us if there is interest in making contributions in the near-term. -Contributors names will be added to [`CONTRIBUTORS.md`](https://github.com/VulcanClimateModeling/fv3core/blob/master/CONTRIBUTORS.md). +pace is actively developed by NOAA/NASA, so please contact us if there is interest in making contributions in the near-term. +Contributors names will be added to [`CONTRIBUTORS.md`](https://github.com/NOAA-GFDL/pace/blob/develop/CONTRIBUTORS.md). ## Linting -Dependencies for linting are maintained in `requirements_lint.txt`, and can be installed with: +We rely on `pre-commit` for linting, which is included in the `[dev]` extra. ```shell -pip install -c constraints.txt -r requirements_lint.txt +pip install -e .[dev] ``` Correcting and checking your code complies with all requirements can be run with: ```shell -make lint +pre-commit run --all-files ``` -We manage the list of syntax requirements using [pre-commit](https://pre-commit.com/). **This runs all checks and is required to pass as one of the continuous integration tests.** The list of checkes includes `black`, `isort`, and `flake8`, among a few others, found in `.pre-commit-config.yaml`. diff --git a/README.md b/README.md index 1159b794..813a14c5 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,22 @@ python -m venv .venv source .venv/bin/activate ``` -Inside of your pace `venv` or `conda` environment, pip install pace (as editable install) with the `lint` and `test` extras: +Inside of your pace `venv` or `conda` environment, install pace and its dependencies. For developers, we recommend an editable install with the `[dev]` extra: ```shell -pip install -e .[lint,test] +pip install -e .[dev] +``` + +For running tests, we recommend to install the `[test]` extra with pinned dependency versions to enable reproducibility: + +```shell +pip install -c constraints.txt .[test] +``` + +For running pace, you don't need any extra: + +```shell +pip install -c constraints.txt . ``` Shell scripts to install Pace on specific machines such as Gaea can be found in `examples/build_scripts/`. @@ -190,3 +202,11 @@ cd pace cp /home/scripts/setup_env.sh . && chmod +x setup_env.sh source ./setup_env.sh ``` + +## Updating constraints + +The `constraints.txt` file can be updated like this: + +```shell +pip-compile --output-file=constraints.txt setup.py +``` diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 00000000..9549f458 --- /dev/null +++ b/constraints.txt @@ -0,0 +1,230 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=constraints.txt setup.py +# +aenum==3.1.16 + # via dace +asciitree==0.3.3 + # via zarr +asttokens==2.4.1 + # via devtools +astunparse==1.6.3 + # via dace +attrs==25.3.0 + # via gt4py +black==25.1.0 + # via gt4py +boltons==25.0.0 + # via gt4py +cached-property==2.0.1 + # via gt4py +certifi==2025.4.26 + # via netcdf4 +cftime==1.6.4.post1 + # via + # ndsl + # netcdf4 +click==8.1.8 + # via + # black + # dask + # gt4py +cloudpickle==3.1.1 + # via dask +cmake==4.0.0 + # via gt4py +contourpy==1.3.2 + # via matplotlib +cycler==0.12.1 + # via matplotlib +cytoolz==1.0.1 + # via gt4py +dace @ file:///home/romanc/code/pace/NDSL/external/dace + # via ndsl +dacite==1.9.2 + # via pace (setup.py) +dask==2025.4.1 + # via ndsl +deepdiff==8.4.2 + # via gt4py +deprecated==1.2.18 + # via numcodecs +devtools==0.12.2 + # via gt4py +dill==0.4.0 + # via dace +executing==2.2.0 + # via devtools +f90nml==1.4.5 + # via + # ndsl + # pace (setup.py) + # pyfv3 + # pyshield +factory-boy==3.3.3 + # via gt4py +faker==37.1.0 + # via factory-boy +fasteners==0.19 + # via zarr +fonttools==4.57.0 + # via matplotlib +fparser==0.2.0 + # via dace +frozendict==2.4.6 + # via gt4py +fsspec==2025.3.2 + # via + # dask + # ndsl +gridtools-cpp==2.3.9 + # via gt4py +gt4py @ file:///home/romanc/code/pace/NDSL/external/gt4py + # via ndsl +h5netcdf==1.6.1 + # via ndsl +h5py==3.13.0 + # via h5netcdf +importlib-metadata==8.7.0 + # via dask +jinja2==3.1.6 + # via gt4py +kiwisolver==1.4.8 + # via matplotlib +lark==1.2.2 + # via gt4py +locket==1.0.0 + # via partd +mako==1.3.10 + # via gt4py +markupsafe==3.0.2 + # via + # jinja2 + # mako +matplotlib==3.10.1 + # via ndsl +mpi4py==3.1.5 + # via ndsl +mpmath==1.3.0 + # via sympy +mypy-extensions==1.1.0 + # via black +nanobind==2.7.0 + # via gt4py +ndsl @ file:///home/romanc/code/pace/NDSL + # via pace (setup.py) +netcdf4==1.7.1 + # via ndsl +networkx==3.4.2 + # via dace +ninja==1.11.1.4 + # via gt4py +numcodecs==0.15.1 + # via zarr +numpy==1.26.4 + # via + # cftime + # contourpy + # dace + # gt4py + # h5py + # matplotlib + # ndsl + # netcdf4 + # numcodecs + # pace (setup.py) + # pandas + # pyfv3 + # pyshield + # scipy + # xarray + # zarr +orderly-set==5.4.0 + # via deepdiff +packaging==25.0 + # via + # black + # dace + # dask + # gt4py + # h5netcdf + # matplotlib + # setuptools-scm + # xarray +pandas==2.2.3 + # via xarray +partd==1.4.2 + # via dask +pathspec==0.12.1 + # via black +pillow==11.2.1 + # via matplotlib +platformdirs==4.3.7 + # via black +ply==3.11 + # via dace +pybind11==2.13.6 + # via gt4py +pyfv3 @ file:///home/romanc/code/pace/pyFV3 + # via pace (setup.py) +pygments==2.19.1 + # via devtools +pyparsing==3.2.3 + # via matplotlib +pyshield @ file:///home/romanc/code/pace/pySHiELD + # via pace (setup.py) +python-dateutil==2.9.0.post0 + # via + # matplotlib + # pandas +pytz==2025.2 + # via pandas +pyyaml==6.0.2 + # via + # dace + # dask +scipy==1.15.2 + # via ndsl +setuptools-scm==8.3.1 + # via fparser +six==1.17.0 + # via + # asttokens + # astunparse + # python-dateutil +sympy==1.14.0 + # via dace +tabulate==0.9.0 + # via gt4py +toolz==1.0.0 + # via + # cytoolz + # dask + # partd +typing-extensions==4.13.2 + # via gt4py +tzdata==2025.2 + # via + # faker + # pandas +wheel==0.45.1 + # via astunparse +wrapt==1.17.2 + # via deprecated +xarray==2025.4.0 + # via + # ndsl + # pace (setup.py) + # pyfv3 + # pyshield +xxhash==3.0.0 + # via gt4py +zarr==2.18.7 + # via pace (setup.py) +zipp==3.21.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/examples/build_scripts/build_gaea_c4.sh b/examples/build_scripts/build_gaea_c4.sh index bffaed7a..6189d1b6 100644 --- a/examples/build_scripts/build_gaea_c4.sh +++ b/examples/build_scripts/build_gaea_c4.sh @@ -23,4 +23,4 @@ conda create -y --name my_name python=3.8 conda activate my_name # install Pace dependencies and its dependencies -pip3 install . +pip3 install -c constraints.txt .[test] diff --git a/examples/build_scripts/build_gaea_c5.sh b/examples/build_scripts/build_gaea_c5.sh index 07dcbcd2..26e5d4b5 100644 --- a/examples/build_scripts/build_gaea_c5.sh +++ b/examples/build_scripts/build_gaea_c5.sh @@ -23,4 +23,4 @@ conda create -y --name my_name python=3.8 conda activate my_name # install the Pace dependencies, GT4Py, and Pace -pip3 install . +pip3 install -c constraints.txt .[test] diff --git a/examples/build_scripts/build_gaea_c5_gcc.sh b/examples/build_scripts/build_gaea_c5_gcc.sh index 61231ff2..3088da67 100644 --- a/examples/build_scripts/build_gaea_c5_gcc.sh +++ b/examples/build_scripts/build_gaea_c5_gcc.sh @@ -32,4 +32,4 @@ conda create -y --name my_name python=3.11.7 conda activate my_name # install Pace dependencies and its dependencies -pip3 install . +pip3 install -c constraints.txt .[test] diff --git a/examples/build_scripts/build_ppan.sh b/examples/build_scripts/build_ppan.sh index 3927d02f..f559a1ca 100644 --- a/examples/build_scripts/build_ppan.sh +++ b/examples/build_scripts/build_ppan.sh @@ -28,7 +28,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.8 matplotlib==3 conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir . +pip3 install --no-cache-dir -c constraints.txt .[test] # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/build_scripts/build_ppan_intel.sh b/examples/build_scripts/build_ppan_intel.sh index c3a78bc8..577fc4d0 100644 --- a/examples/build_scripts/build_ppan_intel.sh +++ b/examples/build_scripts/build_ppan_intel.sh @@ -35,7 +35,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.11.7 matplotlib conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir . +pip3 install --no-cache-dir -c constraints.txt .[test] # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/create_venv.sh b/examples/create_venv.sh index 58e26119..92adf4a4 100755 --- a/examples/create_venv.sh +++ b/examples/create_venv.sh @@ -12,7 +12,7 @@ python3 -m venv venv rundir=$(pwd) cd ${SCRIPT_DIR}/../ -pip3 install -e .[test,lint] +pip3 install -e .[dev] deactivate cd $rundir diff --git a/setup.py b/setup.py index ab7d927d..ab282cb7 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,14 @@ def local_pkg(name: str, relative_path: str) -> str: "pytest", ] -lint_requirements = ["pre-commit"] +dev_requirements = [ + *test_requirements, + "pip-tools", # for pip-compile + "pre-commit", +] extras_require = { - "lint": lint_requirements, + "dev": dev_requirements, "test": test_requirements, } From 794551a80cfc6d07116889e1e1b0198f1566be02 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:57:38 +0200 Subject: [PATCH 09/17] add constraints files for dev and test setup --- .github/workflows/main_unit_tests.yaml | 2 +- README.md | 15 +- constraints-dev.txt | 374 ++++++++++++++++++++ constraints-test.txt | 345 ++++++++++++++++++ examples/build_scripts/build_gaea_c4.sh | 2 +- examples/build_scripts/build_gaea_c5.sh | 2 +- examples/build_scripts/build_gaea_c5_gcc.sh | 2 +- examples/build_scripts/build_ppan.sh | 2 +- examples/build_scripts/build_ppan_intel.sh | 2 +- 9 files changed, 736 insertions(+), 10 deletions(-) create mode 100644 constraints-dev.txt create mode 100644 constraints-test.txt diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index d7a1cb90..624b4178 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -52,7 +52,7 @@ jobs: if: ${{ ! inputs.component_trigger }} run: | cd ${GITHUB_WORKSPACE}/pace - pip install -c constraints.txt .[test] + pip install -c constraints-test.txt . - name: "External trigger: Install packages" if: ${{ inputs.component_trigger }} diff --git a/README.md b/README.md index 813a14c5..e3e8a4e3 100644 --- a/README.md +++ b/README.md @@ -67,16 +67,16 @@ source .venv/bin/activate Inside of your pace `venv` or `conda` environment, install pace and its dependencies. For developers, we recommend an editable install with the `[dev]` extra: ```shell -pip install -e .[dev] +pip install -e -c constraints-dev . ``` -For running tests, we recommend to install the `[test]` extra with pinned dependency versions to enable reproducibility: +For running tests, we recommend to install pace with pinned dependency versions of the test system to enable reproducibility: ```shell -pip install -c constraints.txt .[test] +pip install -c constraints-test.txt . ``` -For running pace, you don't need any extra: +For just running pace, you don't need any extra: ```shell pip install -c constraints.txt . @@ -210,3 +210,10 @@ The `constraints.txt` file can be updated like this: ```shell pip-compile --output-file=constraints.txt setup.py ``` + +Don't forget to also update the extras + +```shell +pip-compile --extra test --output-file=constraints-test.txt setup.py +pip-compile --extra dev --output-file=constraints-dev.txt setup.py +``` diff --git a/constraints-dev.txt b/constraints-dev.txt new file mode 100644 index 00000000..db5fffe0 --- /dev/null +++ b/constraints-dev.txt @@ -0,0 +1,374 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=dev --output-file=constraints-dev.txt setup.py +# +aenum==3.1.16 + # via dace +asciitree==0.3.3 + # via zarr +asttokens==2.4.1 + # via + # devtools + # stack-data +astunparse==1.6.3 + # via dace +attrs==25.3.0 + # via + # gt4py + # jsonschema + # referencing +black==25.1.0 + # via gt4py +boltons==25.0.0 + # via gt4py +build==1.2.2.post1 + # via pip-tools +cached-property==2.0.1 + # via gt4py +certifi==2025.4.26 + # via netcdf4 +cfgv==3.4.0 + # via pre-commit +cftime==1.6.4.post1 + # via + # ndsl + # netcdf4 +click==8.1.8 + # via + # black + # dask + # gt4py + # pip-tools +cloudpickle==3.1.1 + # via dask +cmake==4.0.0 + # via gt4py +comm==0.2.2 + # via ipykernel +contourpy==1.3.2 + # via matplotlib +cycler==0.12.1 + # via matplotlib +cytoolz==1.0.1 + # via gt4py +dace @ file:///home/romanc/code/pace/NDSL/external/dace + # via ndsl +dacite==1.9.2 + # via pace (setup.py) +dask==2025.4.1 + # via ndsl +debugpy==1.8.14 + # via ipykernel +decorator==5.2.1 + # via ipython +deepdiff==8.4.2 + # via gt4py +deprecated==1.2.18 + # via numcodecs +devtools==0.12.2 + # via gt4py +dill==0.4.0 + # via dace +distlib==0.3.9 + # via virtualenv +executing==2.2.0 + # via + # devtools + # stack-data +f90nml==1.4.5 + # via + # ndsl + # pace (setup.py) + # pyfv3 + # pyshield +factory-boy==3.3.3 + # via gt4py +faker==37.1.0 + # via factory-boy +fasteners==0.19 + # via zarr +fastjsonschema==2.21.1 + # via nbformat +filelock==3.18.0 + # via virtualenv +fonttools==4.57.0 + # via matplotlib +fparser==0.2.0 + # via dace +frozendict==2.4.6 + # via gt4py +fsspec==2025.3.2 + # via + # dask + # ndsl +gridtools-cpp==2.3.9 + # via gt4py +gt4py @ file:///home/romanc/code/pace/NDSL/external/gt4py + # via ndsl +h5netcdf==1.6.1 + # via ndsl +h5py==3.13.0 + # via h5netcdf +identify==2.6.10 + # via pre-commit +importlib-metadata==8.7.0 + # via dask +iniconfig==2.1.0 + # via pytest +ipykernel==6.29.5 + # via nbmake +ipython==9.2.0 + # via ipykernel +ipython-pygments-lexers==1.1.1 + # via ipython +jedi==0.19.2 + # via ipython +jinja2==3.1.6 + # via gt4py +jsonschema==4.23.0 + # via nbformat +jsonschema-specifications==2025.4.1 + # via jsonschema +jupyter-client==8.6.3 + # via + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbformat +kiwisolver==1.4.8 + # via matplotlib +lark==1.2.2 + # via gt4py +locket==1.0.0 + # via partd +mako==1.3.10 + # via gt4py +markupsafe==3.0.2 + # via + # jinja2 + # mako +matplotlib==3.10.1 + # via ndsl +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mpi4py==3.1.5 + # via + # ndsl + # pace (setup.py) +mpmath==1.3.0 + # via sympy +mypy-extensions==1.1.0 + # via black +nanobind==2.7.0 + # via gt4py +nbclient==0.10.2 + # via nbmake +nbformat==5.10.4 + # via + # nbclient + # nbmake +nbmake==1.5.5 + # via pace (setup.py) +ndsl @ file:///home/romanc/code/pace/NDSL + # via pace (setup.py) +nest-asyncio==1.6.0 + # via ipykernel +netcdf4==1.7.1 + # via ndsl +networkx==3.4.2 + # via dace +ninja==1.11.1.4 + # via gt4py +nodeenv==1.9.1 + # via pre-commit +numcodecs==0.15.1 + # via zarr +numpy==1.26.4 + # via + # cftime + # contourpy + # dace + # gt4py + # h5py + # matplotlib + # ndsl + # netcdf4 + # numcodecs + # pace (setup.py) + # pandas + # pyfv3 + # pyshield + # scipy + # xarray + # zarr +orderly-set==5.4.0 + # via deepdiff +packaging==25.0 + # via + # black + # build + # dace + # dask + # gt4py + # h5netcdf + # ipykernel + # matplotlib + # pytest + # setuptools-scm + # xarray +pandas==2.2.3 + # via xarray +parso==0.8.4 + # via jedi +partd==1.4.2 + # via dask +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +pillow==11.2.1 + # via matplotlib +pip-tools==7.4.1 + # via pace (setup.py) +platformdirs==4.3.7 + # via + # black + # jupyter-core + # virtualenv +pluggy==1.5.0 + # via pytest +ply==3.11 + # via dace +pre-commit==4.2.0 + # via pace (setup.py) +prompt-toolkit==3.0.51 + # via ipython +psutil==7.0.0 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pybind11==2.13.6 + # via gt4py +pyfv3 @ file:///home/romanc/code/pace/pyFV3 + # via pace (setup.py) +pygments==2.19.1 + # via + # devtools + # ipython + # ipython-pygments-lexers + # nbmake +pyparsing==3.2.3 + # via matplotlib +pyproject-hooks==1.2.0 + # via + # build + # pip-tools +pyshield @ file:///home/romanc/code/pace/pySHiELD + # via pace (setup.py) +pytest==8.3.5 + # via + # nbmake + # pace (setup.py) +python-dateutil==2.9.0.post0 + # via + # jupyter-client + # matplotlib + # pandas +pytz==2025.2 + # via pandas +pyyaml==6.0.2 + # via + # dace + # dask + # pre-commit +pyzmq==26.4.0 + # via + # ipykernel + # jupyter-client +referencing==0.36.2 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.24.0 + # via + # jsonschema + # referencing +scipy==1.15.2 + # via ndsl +setuptools-scm==8.3.1 + # via fparser +six==1.17.0 + # via + # asttokens + # astunparse + # python-dateutil +stack-data==0.6.3 + # via ipython +sympy==1.14.0 + # via dace +tabulate==0.9.0 + # via gt4py +toolz==1.0.0 + # via + # cytoolz + # dask + # partd +tornado==6.4.2 + # via + # ipykernel + # jupyter-client +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbformat +typing-extensions==4.13.2 + # via + # gt4py + # ipython + # referencing +tzdata==2025.2 + # via + # faker + # pandas +virtualenv==20.30.0 + # via pre-commit +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via + # astunparse + # pip-tools +wrapt==1.17.2 + # via deprecated +xarray==2025.4.0 + # via + # ndsl + # pace (setup.py) + # pyfv3 + # pyshield +xxhash==3.0.0 + # via gt4py +zarr==2.18.7 + # via pace (setup.py) +zipp==3.21.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/constraints-test.txt b/constraints-test.txt new file mode 100644 index 00000000..4b8530b7 --- /dev/null +++ b/constraints-test.txt @@ -0,0 +1,345 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --extra=test --output-file=constraints-test.txt setup.py +# +aenum==3.1.16 + # via dace +asciitree==0.3.3 + # via zarr +asttokens==2.4.1 + # via + # devtools + # stack-data +astunparse==1.6.3 + # via dace +attrs==25.3.0 + # via + # gt4py + # jsonschema + # referencing +black==25.1.0 + # via gt4py +boltons==25.0.0 + # via gt4py +cached-property==2.0.1 + # via gt4py +certifi==2025.4.26 + # via netcdf4 +cftime==1.6.4.post1 + # via + # ndsl + # netcdf4 +click==8.1.8 + # via + # black + # dask + # gt4py +cloudpickle==3.1.1 + # via dask +cmake==4.0.0 + # via gt4py +comm==0.2.2 + # via ipykernel +contourpy==1.3.2 + # via matplotlib +cycler==0.12.1 + # via matplotlib +cytoolz==1.0.1 + # via gt4py +dace @ file:///home/romanc/code/pace/NDSL/external/dace + # via ndsl +dacite==1.9.2 + # via pace (setup.py) +dask==2025.4.1 + # via ndsl +debugpy==1.8.14 + # via ipykernel +decorator==5.2.1 + # via ipython +deepdiff==8.4.2 + # via gt4py +deprecated==1.2.18 + # via numcodecs +devtools==0.12.2 + # via gt4py +dill==0.4.0 + # via dace +executing==2.2.0 + # via + # devtools + # stack-data +f90nml==1.4.5 + # via + # ndsl + # pace (setup.py) + # pyfv3 + # pyshield +factory-boy==3.3.3 + # via gt4py +faker==37.1.0 + # via factory-boy +fasteners==0.19 + # via zarr +fastjsonschema==2.21.1 + # via nbformat +fonttools==4.57.0 + # via matplotlib +fparser==0.2.0 + # via dace +frozendict==2.4.6 + # via gt4py +fsspec==2025.3.2 + # via + # dask + # ndsl +gridtools-cpp==2.3.9 + # via gt4py +gt4py @ file:///home/romanc/code/pace/NDSL/external/gt4py + # via ndsl +h5netcdf==1.6.1 + # via ndsl +h5py==3.13.0 + # via h5netcdf +importlib-metadata==8.7.0 + # via dask +iniconfig==2.1.0 + # via pytest +ipykernel==6.29.5 + # via nbmake +ipython==9.2.0 + # via ipykernel +ipython-pygments-lexers==1.1.1 + # via ipython +jedi==0.19.2 + # via ipython +jinja2==3.1.6 + # via gt4py +jsonschema==4.23.0 + # via nbformat +jsonschema-specifications==2025.4.1 + # via jsonschema +jupyter-client==8.6.3 + # via + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbformat +kiwisolver==1.4.8 + # via matplotlib +lark==1.2.2 + # via gt4py +locket==1.0.0 + # via partd +mako==1.3.10 + # via gt4py +markupsafe==3.0.2 + # via + # jinja2 + # mako +matplotlib==3.10.1 + # via ndsl +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mpi4py==3.1.5 + # via + # ndsl + # pace (setup.py) +mpmath==1.3.0 + # via sympy +mypy-extensions==1.1.0 + # via black +nanobind==2.7.0 + # via gt4py +nbclient==0.10.2 + # via nbmake +nbformat==5.10.4 + # via + # nbclient + # nbmake +nbmake==1.5.5 + # via pace (setup.py) +ndsl @ file:///home/romanc/code/pace/NDSL + # via pace (setup.py) +nest-asyncio==1.6.0 + # via ipykernel +netcdf4==1.7.1 + # via ndsl +networkx==3.4.2 + # via dace +ninja==1.11.1.4 + # via gt4py +numcodecs==0.15.1 + # via zarr +numpy==1.26.4 + # via + # cftime + # contourpy + # dace + # gt4py + # h5py + # matplotlib + # ndsl + # netcdf4 + # numcodecs + # pace (setup.py) + # pandas + # pyfv3 + # pyshield + # scipy + # xarray + # zarr +orderly-set==5.4.0 + # via deepdiff +packaging==25.0 + # via + # black + # dace + # dask + # gt4py + # h5netcdf + # ipykernel + # matplotlib + # pytest + # setuptools-scm + # xarray +pandas==2.2.3 + # via xarray +parso==0.8.4 + # via jedi +partd==1.4.2 + # via dask +pathspec==0.12.1 + # via black +pexpect==4.9.0 + # via ipython +pillow==11.2.1 + # via matplotlib +platformdirs==4.3.7 + # via + # black + # jupyter-core +pluggy==1.5.0 + # via pytest +ply==3.11 + # via dace +prompt-toolkit==3.0.51 + # via ipython +psutil==7.0.0 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pybind11==2.13.6 + # via gt4py +pyfv3 @ file:///home/romanc/code/pace/pyFV3 + # via pace (setup.py) +pygments==2.19.1 + # via + # devtools + # ipython + # ipython-pygments-lexers + # nbmake +pyparsing==3.2.3 + # via matplotlib +pyshield @ file:///home/romanc/code/pace/pySHiELD + # via pace (setup.py) +pytest==8.3.5 + # via + # nbmake + # pace (setup.py) +python-dateutil==2.9.0.post0 + # via + # jupyter-client + # matplotlib + # pandas +pytz==2025.2 + # via pandas +pyyaml==6.0.2 + # via + # dace + # dask +pyzmq==26.4.0 + # via + # ipykernel + # jupyter-client +referencing==0.36.2 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.24.0 + # via + # jsonschema + # referencing +scipy==1.15.2 + # via ndsl +setuptools-scm==8.3.1 + # via fparser +six==1.17.0 + # via + # asttokens + # astunparse + # python-dateutil +stack-data==0.6.3 + # via ipython +sympy==1.14.0 + # via dace +tabulate==0.9.0 + # via gt4py +toolz==1.0.0 + # via + # cytoolz + # dask + # partd +tornado==6.4.2 + # via + # ipykernel + # jupyter-client +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbformat +typing-extensions==4.13.2 + # via + # gt4py + # ipython + # referencing +tzdata==2025.2 + # via + # faker + # pandas +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.45.1 + # via astunparse +wrapt==1.17.2 + # via deprecated +xarray==2025.4.0 + # via + # ndsl + # pace (setup.py) + # pyfv3 + # pyshield +xxhash==3.0.0 + # via gt4py +zarr==2.18.7 + # via pace (setup.py) +zipp==3.21.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/examples/build_scripts/build_gaea_c4.sh b/examples/build_scripts/build_gaea_c4.sh index 6189d1b6..0c8c134f 100644 --- a/examples/build_scripts/build_gaea_c4.sh +++ b/examples/build_scripts/build_gaea_c4.sh @@ -23,4 +23,4 @@ conda create -y --name my_name python=3.8 conda activate my_name # install Pace dependencies and its dependencies -pip3 install -c constraints.txt .[test] +pip3 install -c constraints-test.txt . diff --git a/examples/build_scripts/build_gaea_c5.sh b/examples/build_scripts/build_gaea_c5.sh index 26e5d4b5..0fa8e369 100644 --- a/examples/build_scripts/build_gaea_c5.sh +++ b/examples/build_scripts/build_gaea_c5.sh @@ -23,4 +23,4 @@ conda create -y --name my_name python=3.8 conda activate my_name # install the Pace dependencies, GT4Py, and Pace -pip3 install -c constraints.txt .[test] +pip3 install -c constraints-test.txt . diff --git a/examples/build_scripts/build_gaea_c5_gcc.sh b/examples/build_scripts/build_gaea_c5_gcc.sh index 3088da67..7b465222 100644 --- a/examples/build_scripts/build_gaea_c5_gcc.sh +++ b/examples/build_scripts/build_gaea_c5_gcc.sh @@ -32,4 +32,4 @@ conda create -y --name my_name python=3.11.7 conda activate my_name # install Pace dependencies and its dependencies -pip3 install -c constraints.txt .[test] +pip3 install -c constraints-test.txt . diff --git a/examples/build_scripts/build_ppan.sh b/examples/build_scripts/build_ppan.sh index f559a1ca..b1af3ec5 100644 --- a/examples/build_scripts/build_ppan.sh +++ b/examples/build_scripts/build_ppan.sh @@ -28,7 +28,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.8 matplotlib==3 conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir -c constraints.txt .[test] +pip3 install --no-cache-dir -c constraints-test.txt . # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/build_scripts/build_ppan_intel.sh b/examples/build_scripts/build_ppan_intel.sh index 577fc4d0..9f2e41e6 100644 --- a/examples/build_scripts/build_ppan_intel.sh +++ b/examples/build_scripts/build_ppan_intel.sh @@ -35,7 +35,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.11.7 matplotlib conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir -c constraints.txt .[test] +pip3 install --no-cache-dir -c constraints-test.txt . # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR From c8b51ea3cbbd970e264fa2bfba9aa42288f4fac4 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:03:08 +0200 Subject: [PATCH 10/17] fix typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e3e8a4e3..daa594d3 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ source .venv/bin/activate Inside of your pace `venv` or `conda` environment, install pace and its dependencies. For developers, we recommend an editable install with the `[dev]` extra: ```shell -pip install -e -c constraints-dev . +pip install -c constraints-dev.txt -e . ``` For running tests, we recommend to install pace with pinned dependency versions of the test system to enable reproducibility: From 9b473b43ec31479684032d0ef2d96141ee7a95f9 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:29:53 +0200 Subject: [PATCH 11/17] Remove hard-coded paths from constraints file --- README.md | 2 ++ constraints-dev.txt | 10 ---------- constraints-test.txt | 10 ---------- constraints.txt | 10 ---------- 4 files changed, 2 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index daa594d3..72bb5a69 100644 --- a/README.md +++ b/README.md @@ -217,3 +217,5 @@ Don't forget to also update the extras pip-compile --extra test --output-file=constraints-test.txt setup.py pip-compile --extra dev --output-file=constraints-dev.txt setup.py ``` + +Once updated, manually remove the packages with hard-coded paths (e.g. `dace`, `gt4py`, `ndsl`, `pyfv3`, and `pyshield`). CI won't run otherwise (because the paths are different) and since we are installing them from a submodule it is safe to not further pin them (they won't change unless the submodule changes, which - in turn - is versioned through `git`). diff --git a/constraints-dev.txt b/constraints-dev.txt index db5fffe0..9a2d12f7 100644 --- a/constraints-dev.txt +++ b/constraints-dev.txt @@ -53,8 +53,6 @@ cycler==0.12.1 # via matplotlib cytoolz==1.0.1 # via gt4py -dace @ file:///home/romanc/code/pace/NDSL/external/dace - # via ndsl dacite==1.9.2 # via pace (setup.py) dask==2025.4.1 @@ -105,8 +103,6 @@ fsspec==2025.3.2 # ndsl gridtools-cpp==2.3.9 # via gt4py -gt4py @ file:///home/romanc/code/pace/NDSL/external/gt4py - # via ndsl h5netcdf==1.6.1 # via ndsl h5py==3.13.0 @@ -177,8 +173,6 @@ nbformat==5.10.4 # nbmake nbmake==1.5.5 # via pace (setup.py) -ndsl @ file:///home/romanc/code/pace/NDSL - # via pace (setup.py) nest-asyncio==1.6.0 # via ipykernel netcdf4==1.7.1 @@ -259,8 +253,6 @@ pure-eval==0.2.3 # via stack-data pybind11==2.13.6 # via gt4py -pyfv3 @ file:///home/romanc/code/pace/pyFV3 - # via pace (setup.py) pygments==2.19.1 # via # devtools @@ -273,8 +265,6 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pyshield @ file:///home/romanc/code/pace/pySHiELD - # via pace (setup.py) pytest==8.3.5 # via # nbmake diff --git a/constraints-test.txt b/constraints-test.txt index 4b8530b7..5982e4ae 100644 --- a/constraints-test.txt +++ b/constraints-test.txt @@ -48,8 +48,6 @@ cycler==0.12.1 # via matplotlib cytoolz==1.0.1 # via gt4py -dace @ file:///home/romanc/code/pace/NDSL/external/dace - # via ndsl dacite==1.9.2 # via pace (setup.py) dask==2025.4.1 @@ -96,8 +94,6 @@ fsspec==2025.3.2 # ndsl gridtools-cpp==2.3.9 # via gt4py -gt4py @ file:///home/romanc/code/pace/NDSL/external/gt4py - # via ndsl h5netcdf==1.6.1 # via ndsl h5py==3.13.0 @@ -166,8 +162,6 @@ nbformat==5.10.4 # nbmake nbmake==1.5.5 # via pace (setup.py) -ndsl @ file:///home/romanc/code/pace/NDSL - # via pace (setup.py) nest-asyncio==1.6.0 # via ipykernel netcdf4==1.7.1 @@ -240,8 +234,6 @@ pure-eval==0.2.3 # via stack-data pybind11==2.13.6 # via gt4py -pyfv3 @ file:///home/romanc/code/pace/pyFV3 - # via pace (setup.py) pygments==2.19.1 # via # devtools @@ -250,8 +242,6 @@ pygments==2.19.1 # nbmake pyparsing==3.2.3 # via matplotlib -pyshield @ file:///home/romanc/code/pace/pySHiELD - # via pace (setup.py) pytest==8.3.5 # via # nbmake diff --git a/constraints.txt b/constraints.txt index 9549f458..de5991aa 100644 --- a/constraints.txt +++ b/constraints.txt @@ -41,8 +41,6 @@ cycler==0.12.1 # via matplotlib cytoolz==1.0.1 # via gt4py -dace @ file:///home/romanc/code/pace/NDSL/external/dace - # via ndsl dacite==1.9.2 # via pace (setup.py) dask==2025.4.1 @@ -81,8 +79,6 @@ fsspec==2025.3.2 # ndsl gridtools-cpp==2.3.9 # via gt4py -gt4py @ file:///home/romanc/code/pace/NDSL/external/gt4py - # via ndsl h5netcdf==1.6.1 # via ndsl h5py==3.13.0 @@ -113,8 +109,6 @@ mypy-extensions==1.1.0 # via black nanobind==2.7.0 # via gt4py -ndsl @ file:///home/romanc/code/pace/NDSL - # via pace (setup.py) netcdf4==1.7.1 # via ndsl networkx==3.4.2 @@ -167,14 +161,10 @@ ply==3.11 # via dace pybind11==2.13.6 # via gt4py -pyfv3 @ file:///home/romanc/code/pace/pyFV3 - # via pace (setup.py) pygments==2.19.1 # via devtools pyparsing==3.2.3 # via matplotlib -pyshield @ file:///home/romanc/code/pace/pySHiELD - # via pace (setup.py) python-dateutil==2.9.0.post0 # via # matplotlib From 5d381a22d9af8cd84469c5fe00f4474cb459cd3d Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:36:11 +0200 Subject: [PATCH 12/17] Keep docs/scripts consistent --- CONTRIBUTING.md | 2 +- examples/create_venv.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3205f125..32b32372 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ Contributors names will be added to [`CONTRIBUTORS.md`](https://github.com/NOAA- We rely on `pre-commit` for linting, which is included in the `[dev]` extra. ```shell -pip install -e .[dev] +pip install -c constraints-dev.txt -e . ``` Correcting and checking your code complies with all requirements can be run with: diff --git a/examples/create_venv.sh b/examples/create_venv.sh index 92adf4a4..68c197e5 100755 --- a/examples/create_venv.sh +++ b/examples/create_venv.sh @@ -12,7 +12,7 @@ python3 -m venv venv rundir=$(pwd) cd ${SCRIPT_DIR}/../ -pip3 install -e .[dev] +pip install -c constraints-dev.txt -e . deactivate cd $rundir From e57783b8b22545166ced333721b126f0f9ff11e5 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:37:03 +0200 Subject: [PATCH 13/17] no need for pip list anymore (we have constraints file now) --- .github/workflows/main_unit_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index 624b4178..c6385c58 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -46,7 +46,6 @@ jobs: run: | python --version pip --version - pip list - name: Install packages if: ${{ ! inputs.component_trigger }} From 0e8c5478fda52d7d76d7a75953f587a16ac92eb2 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:43:02 +0200 Subject: [PATCH 14/17] actually install extras ;) --- .github/workflows/main_unit_tests.yaml | 2 +- README.md | 4 ++-- examples/build_scripts/build_gaea_c4.sh | 2 +- examples/build_scripts/build_gaea_c5.sh | 2 +- examples/build_scripts/build_gaea_c5_gcc.sh | 2 +- examples/build_scripts/build_ppan.sh | 2 +- examples/build_scripts/build_ppan_intel.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index c6385c58..2ea8ff8f 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -51,7 +51,7 @@ jobs: if: ${{ ! inputs.component_trigger }} run: | cd ${GITHUB_WORKSPACE}/pace - pip install -c constraints-test.txt . + pip install -c constraints-test.txt .[test] - name: "External trigger: Install packages" if: ${{ inputs.component_trigger }} diff --git a/README.md b/README.md index 72bb5a69..cb616cd6 100644 --- a/README.md +++ b/README.md @@ -67,13 +67,13 @@ source .venv/bin/activate Inside of your pace `venv` or `conda` environment, install pace and its dependencies. For developers, we recommend an editable install with the `[dev]` extra: ```shell -pip install -c constraints-dev.txt -e . +pip install -c constraints-dev.txt -e .[dev] ``` For running tests, we recommend to install pace with pinned dependency versions of the test system to enable reproducibility: ```shell -pip install -c constraints-test.txt . +pip install -c constraints-test.txt .[test] ``` For just running pace, you don't need any extra: diff --git a/examples/build_scripts/build_gaea_c4.sh b/examples/build_scripts/build_gaea_c4.sh index 0c8c134f..55ca940a 100644 --- a/examples/build_scripts/build_gaea_c4.sh +++ b/examples/build_scripts/build_gaea_c4.sh @@ -23,4 +23,4 @@ conda create -y --name my_name python=3.8 conda activate my_name # install Pace dependencies and its dependencies -pip3 install -c constraints-test.txt . +pip3 install -c constraints-test.txt .[test] diff --git a/examples/build_scripts/build_gaea_c5.sh b/examples/build_scripts/build_gaea_c5.sh index 0fa8e369..73c4c2b8 100644 --- a/examples/build_scripts/build_gaea_c5.sh +++ b/examples/build_scripts/build_gaea_c5.sh @@ -23,4 +23,4 @@ conda create -y --name my_name python=3.8 conda activate my_name # install the Pace dependencies, GT4Py, and Pace -pip3 install -c constraints-test.txt . +pip3 install -c constraints-test.txt .[test] diff --git a/examples/build_scripts/build_gaea_c5_gcc.sh b/examples/build_scripts/build_gaea_c5_gcc.sh index 7b465222..aa84c826 100644 --- a/examples/build_scripts/build_gaea_c5_gcc.sh +++ b/examples/build_scripts/build_gaea_c5_gcc.sh @@ -32,4 +32,4 @@ conda create -y --name my_name python=3.11.7 conda activate my_name # install Pace dependencies and its dependencies -pip3 install -c constraints-test.txt . +pip3 install -c constraints-test.txt .[test] diff --git a/examples/build_scripts/build_ppan.sh b/examples/build_scripts/build_ppan.sh index b1af3ec5..8c112e2d 100644 --- a/examples/build_scripts/build_ppan.sh +++ b/examples/build_scripts/build_ppan.sh @@ -28,7 +28,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.8 matplotlib==3 conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir -c constraints-test.txt . +pip3 install --no-cache-dir -c constraints-test.txt .[test] # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/build_scripts/build_ppan_intel.sh b/examples/build_scripts/build_ppan_intel.sh index 9f2e41e6..d8a81e4b 100644 --- a/examples/build_scripts/build_ppan_intel.sh +++ b/examples/build_scripts/build_ppan_intel.sh @@ -35,7 +35,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.11.7 matplotlib conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir -c constraints-test.txt . +pip3 install --no-cache-dir -c constraints-test.txt .[test] # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR From 8bb9d6d6524f2c26061925d9f11e4a399ce4f5bd Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:07:54 +0200 Subject: [PATCH 15/17] Remove constraints* files again --- .github/workflows/main_unit_tests.yaml | 11 +- CONTRIBUTING.md | 2 +- README.md | 23 +- constraints-dev.txt | 364 -------------------- constraints-test.txt | 335 ------------------ constraints.txt | 220 ------------ examples/build_scripts/build_gaea_c4.sh | 2 +- examples/build_scripts/build_gaea_c5.sh | 4 +- examples/build_scripts/build_gaea_c5_gcc.sh | 2 +- examples/build_scripts/build_ppan.sh | 2 +- examples/build_scripts/build_ppan_intel.sh | 2 +- examples/create_venv.sh | 2 +- 12 files changed, 12 insertions(+), 957 deletions(-) delete mode 100644 constraints-dev.txt delete mode 100644 constraints-test.txt delete mode 100644 constraints.txt diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index 2ea8ff8f..fe214c89 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -48,18 +48,9 @@ jobs: pip --version - name: Install packages - if: ${{ ! inputs.component_trigger }} run: | cd ${GITHUB_WORKSPACE}/pace - pip install -c constraints-test.txt .[test] - - - name: "External trigger: Install packages" - if: ${{ inputs.component_trigger }} - # Ignore `constraints.txt` when running as part of the NDSL pipeline - # to avoid false positives in package dependency resolution. - run: | - cd ${GITHUB_WORKSPACE}/pace - pip3 install .[test] + pip install .[test] - name: Prepare input files run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32b32372..3205f125 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ Contributors names will be added to [`CONTRIBUTORS.md`](https://github.com/NOAA- We rely on `pre-commit` for linting, which is included in the `[dev]` extra. ```shell -pip install -c constraints-dev.txt -e . +pip install -e .[dev] ``` Correcting and checking your code complies with all requirements can be run with: diff --git a/README.md b/README.md index cb616cd6..4e9110db 100644 --- a/README.md +++ b/README.md @@ -67,19 +67,19 @@ source .venv/bin/activate Inside of your pace `venv` or `conda` environment, install pace and its dependencies. For developers, we recommend an editable install with the `[dev]` extra: ```shell -pip install -c constraints-dev.txt -e .[dev] +pip install -e .[dev] ``` For running tests, we recommend to install pace with pinned dependency versions of the test system to enable reproducibility: ```shell -pip install -c constraints-test.txt .[test] +pip install .[test] ``` For just running pace, you don't need any extra: ```shell -pip install -c constraints.txt . +pip install . ``` Shell scripts to install Pace on specific machines such as Gaea can be found in `examples/build_scripts/`. @@ -202,20 +202,3 @@ cd pace cp /home/scripts/setup_env.sh . && chmod +x setup_env.sh source ./setup_env.sh ``` - -## Updating constraints - -The `constraints.txt` file can be updated like this: - -```shell -pip-compile --output-file=constraints.txt setup.py -``` - -Don't forget to also update the extras - -```shell -pip-compile --extra test --output-file=constraints-test.txt setup.py -pip-compile --extra dev --output-file=constraints-dev.txt setup.py -``` - -Once updated, manually remove the packages with hard-coded paths (e.g. `dace`, `gt4py`, `ndsl`, `pyfv3`, and `pyshield`). CI won't run otherwise (because the paths are different) and since we are installing them from a submodule it is safe to not further pin them (they won't change unless the submodule changes, which - in turn - is versioned through `git`). diff --git a/constraints-dev.txt b/constraints-dev.txt deleted file mode 100644 index 9a2d12f7..00000000 --- a/constraints-dev.txt +++ /dev/null @@ -1,364 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=dev --output-file=constraints-dev.txt setup.py -# -aenum==3.1.16 - # via dace -asciitree==0.3.3 - # via zarr -asttokens==2.4.1 - # via - # devtools - # stack-data -astunparse==1.6.3 - # via dace -attrs==25.3.0 - # via - # gt4py - # jsonschema - # referencing -black==25.1.0 - # via gt4py -boltons==25.0.0 - # via gt4py -build==1.2.2.post1 - # via pip-tools -cached-property==2.0.1 - # via gt4py -certifi==2025.4.26 - # via netcdf4 -cfgv==3.4.0 - # via pre-commit -cftime==1.6.4.post1 - # via - # ndsl - # netcdf4 -click==8.1.8 - # via - # black - # dask - # gt4py - # pip-tools -cloudpickle==3.1.1 - # via dask -cmake==4.0.0 - # via gt4py -comm==0.2.2 - # via ipykernel -contourpy==1.3.2 - # via matplotlib -cycler==0.12.1 - # via matplotlib -cytoolz==1.0.1 - # via gt4py -dacite==1.9.2 - # via pace (setup.py) -dask==2025.4.1 - # via ndsl -debugpy==1.8.14 - # via ipykernel -decorator==5.2.1 - # via ipython -deepdiff==8.4.2 - # via gt4py -deprecated==1.2.18 - # via numcodecs -devtools==0.12.2 - # via gt4py -dill==0.4.0 - # via dace -distlib==0.3.9 - # via virtualenv -executing==2.2.0 - # via - # devtools - # stack-data -f90nml==1.4.5 - # via - # ndsl - # pace (setup.py) - # pyfv3 - # pyshield -factory-boy==3.3.3 - # via gt4py -faker==37.1.0 - # via factory-boy -fasteners==0.19 - # via zarr -fastjsonschema==2.21.1 - # via nbformat -filelock==3.18.0 - # via virtualenv -fonttools==4.57.0 - # via matplotlib -fparser==0.2.0 - # via dace -frozendict==2.4.6 - # via gt4py -fsspec==2025.3.2 - # via - # dask - # ndsl -gridtools-cpp==2.3.9 - # via gt4py -h5netcdf==1.6.1 - # via ndsl -h5py==3.13.0 - # via h5netcdf -identify==2.6.10 - # via pre-commit -importlib-metadata==8.7.0 - # via dask -iniconfig==2.1.0 - # via pytest -ipykernel==6.29.5 - # via nbmake -ipython==9.2.0 - # via ipykernel -ipython-pygments-lexers==1.1.1 - # via ipython -jedi==0.19.2 - # via ipython -jinja2==3.1.6 - # via gt4py -jsonschema==4.23.0 - # via nbformat -jsonschema-specifications==2025.4.1 - # via jsonschema -jupyter-client==8.6.3 - # via - # ipykernel - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbclient - # nbformat -kiwisolver==1.4.8 - # via matplotlib -lark==1.2.2 - # via gt4py -locket==1.0.0 - # via partd -mako==1.3.10 - # via gt4py -markupsafe==3.0.2 - # via - # jinja2 - # mako -matplotlib==3.10.1 - # via ndsl -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mpi4py==3.1.5 - # via - # ndsl - # pace (setup.py) -mpmath==1.3.0 - # via sympy -mypy-extensions==1.1.0 - # via black -nanobind==2.7.0 - # via gt4py -nbclient==0.10.2 - # via nbmake -nbformat==5.10.4 - # via - # nbclient - # nbmake -nbmake==1.5.5 - # via pace (setup.py) -nest-asyncio==1.6.0 - # via ipykernel -netcdf4==1.7.1 - # via ndsl -networkx==3.4.2 - # via dace -ninja==1.11.1.4 - # via gt4py -nodeenv==1.9.1 - # via pre-commit -numcodecs==0.15.1 - # via zarr -numpy==1.26.4 - # via - # cftime - # contourpy - # dace - # gt4py - # h5py - # matplotlib - # ndsl - # netcdf4 - # numcodecs - # pace (setup.py) - # pandas - # pyfv3 - # pyshield - # scipy - # xarray - # zarr -orderly-set==5.4.0 - # via deepdiff -packaging==25.0 - # via - # black - # build - # dace - # dask - # gt4py - # h5netcdf - # ipykernel - # matplotlib - # pytest - # setuptools-scm - # xarray -pandas==2.2.3 - # via xarray -parso==0.8.4 - # via jedi -partd==1.4.2 - # via dask -pathspec==0.12.1 - # via black -pexpect==4.9.0 - # via ipython -pillow==11.2.1 - # via matplotlib -pip-tools==7.4.1 - # via pace (setup.py) -platformdirs==4.3.7 - # via - # black - # jupyter-core - # virtualenv -pluggy==1.5.0 - # via pytest -ply==3.11 - # via dace -pre-commit==4.2.0 - # via pace (setup.py) -prompt-toolkit==3.0.51 - # via ipython -psutil==7.0.0 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.3 - # via stack-data -pybind11==2.13.6 - # via gt4py -pygments==2.19.1 - # via - # devtools - # ipython - # ipython-pygments-lexers - # nbmake -pyparsing==3.2.3 - # via matplotlib -pyproject-hooks==1.2.0 - # via - # build - # pip-tools -pytest==8.3.5 - # via - # nbmake - # pace (setup.py) -python-dateutil==2.9.0.post0 - # via - # jupyter-client - # matplotlib - # pandas -pytz==2025.2 - # via pandas -pyyaml==6.0.2 - # via - # dace - # dask - # pre-commit -pyzmq==26.4.0 - # via - # ipykernel - # jupyter-client -referencing==0.36.2 - # via - # jsonschema - # jsonschema-specifications -rpds-py==0.24.0 - # via - # jsonschema - # referencing -scipy==1.15.2 - # via ndsl -setuptools-scm==8.3.1 - # via fparser -six==1.17.0 - # via - # asttokens - # astunparse - # python-dateutil -stack-data==0.6.3 - # via ipython -sympy==1.14.0 - # via dace -tabulate==0.9.0 - # via gt4py -toolz==1.0.0 - # via - # cytoolz - # dask - # partd -tornado==6.4.2 - # via - # ipykernel - # jupyter-client -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbformat -typing-extensions==4.13.2 - # via - # gt4py - # ipython - # referencing -tzdata==2025.2 - # via - # faker - # pandas -virtualenv==20.30.0 - # via pre-commit -wcwidth==0.2.13 - # via prompt-toolkit -wheel==0.45.1 - # via - # astunparse - # pip-tools -wrapt==1.17.2 - # via deprecated -xarray==2025.4.0 - # via - # ndsl - # pace (setup.py) - # pyfv3 - # pyshield -xxhash==3.0.0 - # via gt4py -zarr==2.18.7 - # via pace (setup.py) -zipp==3.21.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/constraints-test.txt b/constraints-test.txt deleted file mode 100644 index 5982e4ae..00000000 --- a/constraints-test.txt +++ /dev/null @@ -1,335 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=test --output-file=constraints-test.txt setup.py -# -aenum==3.1.16 - # via dace -asciitree==0.3.3 - # via zarr -asttokens==2.4.1 - # via - # devtools - # stack-data -astunparse==1.6.3 - # via dace -attrs==25.3.0 - # via - # gt4py - # jsonschema - # referencing -black==25.1.0 - # via gt4py -boltons==25.0.0 - # via gt4py -cached-property==2.0.1 - # via gt4py -certifi==2025.4.26 - # via netcdf4 -cftime==1.6.4.post1 - # via - # ndsl - # netcdf4 -click==8.1.8 - # via - # black - # dask - # gt4py -cloudpickle==3.1.1 - # via dask -cmake==4.0.0 - # via gt4py -comm==0.2.2 - # via ipykernel -contourpy==1.3.2 - # via matplotlib -cycler==0.12.1 - # via matplotlib -cytoolz==1.0.1 - # via gt4py -dacite==1.9.2 - # via pace (setup.py) -dask==2025.4.1 - # via ndsl -debugpy==1.8.14 - # via ipykernel -decorator==5.2.1 - # via ipython -deepdiff==8.4.2 - # via gt4py -deprecated==1.2.18 - # via numcodecs -devtools==0.12.2 - # via gt4py -dill==0.4.0 - # via dace -executing==2.2.0 - # via - # devtools - # stack-data -f90nml==1.4.5 - # via - # ndsl - # pace (setup.py) - # pyfv3 - # pyshield -factory-boy==3.3.3 - # via gt4py -faker==37.1.0 - # via factory-boy -fasteners==0.19 - # via zarr -fastjsonschema==2.21.1 - # via nbformat -fonttools==4.57.0 - # via matplotlib -fparser==0.2.0 - # via dace -frozendict==2.4.6 - # via gt4py -fsspec==2025.3.2 - # via - # dask - # ndsl -gridtools-cpp==2.3.9 - # via gt4py -h5netcdf==1.6.1 - # via ndsl -h5py==3.13.0 - # via h5netcdf -importlib-metadata==8.7.0 - # via dask -iniconfig==2.1.0 - # via pytest -ipykernel==6.29.5 - # via nbmake -ipython==9.2.0 - # via ipykernel -ipython-pygments-lexers==1.1.1 - # via ipython -jedi==0.19.2 - # via ipython -jinja2==3.1.6 - # via gt4py -jsonschema==4.23.0 - # via nbformat -jsonschema-specifications==2025.4.1 - # via jsonschema -jupyter-client==8.6.3 - # via - # ipykernel - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbclient - # nbformat -kiwisolver==1.4.8 - # via matplotlib -lark==1.2.2 - # via gt4py -locket==1.0.0 - # via partd -mako==1.3.10 - # via gt4py -markupsafe==3.0.2 - # via - # jinja2 - # mako -matplotlib==3.10.1 - # via ndsl -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mpi4py==3.1.5 - # via - # ndsl - # pace (setup.py) -mpmath==1.3.0 - # via sympy -mypy-extensions==1.1.0 - # via black -nanobind==2.7.0 - # via gt4py -nbclient==0.10.2 - # via nbmake -nbformat==5.10.4 - # via - # nbclient - # nbmake -nbmake==1.5.5 - # via pace (setup.py) -nest-asyncio==1.6.0 - # via ipykernel -netcdf4==1.7.1 - # via ndsl -networkx==3.4.2 - # via dace -ninja==1.11.1.4 - # via gt4py -numcodecs==0.15.1 - # via zarr -numpy==1.26.4 - # via - # cftime - # contourpy - # dace - # gt4py - # h5py - # matplotlib - # ndsl - # netcdf4 - # numcodecs - # pace (setup.py) - # pandas - # pyfv3 - # pyshield - # scipy - # xarray - # zarr -orderly-set==5.4.0 - # via deepdiff -packaging==25.0 - # via - # black - # dace - # dask - # gt4py - # h5netcdf - # ipykernel - # matplotlib - # pytest - # setuptools-scm - # xarray -pandas==2.2.3 - # via xarray -parso==0.8.4 - # via jedi -partd==1.4.2 - # via dask -pathspec==0.12.1 - # via black -pexpect==4.9.0 - # via ipython -pillow==11.2.1 - # via matplotlib -platformdirs==4.3.7 - # via - # black - # jupyter-core -pluggy==1.5.0 - # via pytest -ply==3.11 - # via dace -prompt-toolkit==3.0.51 - # via ipython -psutil==7.0.0 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.3 - # via stack-data -pybind11==2.13.6 - # via gt4py -pygments==2.19.1 - # via - # devtools - # ipython - # ipython-pygments-lexers - # nbmake -pyparsing==3.2.3 - # via matplotlib -pytest==8.3.5 - # via - # nbmake - # pace (setup.py) -python-dateutil==2.9.0.post0 - # via - # jupyter-client - # matplotlib - # pandas -pytz==2025.2 - # via pandas -pyyaml==6.0.2 - # via - # dace - # dask -pyzmq==26.4.0 - # via - # ipykernel - # jupyter-client -referencing==0.36.2 - # via - # jsonschema - # jsonschema-specifications -rpds-py==0.24.0 - # via - # jsonschema - # referencing -scipy==1.15.2 - # via ndsl -setuptools-scm==8.3.1 - # via fparser -six==1.17.0 - # via - # asttokens - # astunparse - # python-dateutil -stack-data==0.6.3 - # via ipython -sympy==1.14.0 - # via dace -tabulate==0.9.0 - # via gt4py -toolz==1.0.0 - # via - # cytoolz - # dask - # partd -tornado==6.4.2 - # via - # ipykernel - # jupyter-client -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbformat -typing-extensions==4.13.2 - # via - # gt4py - # ipython - # referencing -tzdata==2025.2 - # via - # faker - # pandas -wcwidth==0.2.13 - # via prompt-toolkit -wheel==0.45.1 - # via astunparse -wrapt==1.17.2 - # via deprecated -xarray==2025.4.0 - # via - # ndsl - # pace (setup.py) - # pyfv3 - # pyshield -xxhash==3.0.0 - # via gt4py -zarr==2.18.7 - # via pace (setup.py) -zipp==3.21.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index de5991aa..00000000 --- a/constraints.txt +++ /dev/null @@ -1,220 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --output-file=constraints.txt setup.py -# -aenum==3.1.16 - # via dace -asciitree==0.3.3 - # via zarr -asttokens==2.4.1 - # via devtools -astunparse==1.6.3 - # via dace -attrs==25.3.0 - # via gt4py -black==25.1.0 - # via gt4py -boltons==25.0.0 - # via gt4py -cached-property==2.0.1 - # via gt4py -certifi==2025.4.26 - # via netcdf4 -cftime==1.6.4.post1 - # via - # ndsl - # netcdf4 -click==8.1.8 - # via - # black - # dask - # gt4py -cloudpickle==3.1.1 - # via dask -cmake==4.0.0 - # via gt4py -contourpy==1.3.2 - # via matplotlib -cycler==0.12.1 - # via matplotlib -cytoolz==1.0.1 - # via gt4py -dacite==1.9.2 - # via pace (setup.py) -dask==2025.4.1 - # via ndsl -deepdiff==8.4.2 - # via gt4py -deprecated==1.2.18 - # via numcodecs -devtools==0.12.2 - # via gt4py -dill==0.4.0 - # via dace -executing==2.2.0 - # via devtools -f90nml==1.4.5 - # via - # ndsl - # pace (setup.py) - # pyfv3 - # pyshield -factory-boy==3.3.3 - # via gt4py -faker==37.1.0 - # via factory-boy -fasteners==0.19 - # via zarr -fonttools==4.57.0 - # via matplotlib -fparser==0.2.0 - # via dace -frozendict==2.4.6 - # via gt4py -fsspec==2025.3.2 - # via - # dask - # ndsl -gridtools-cpp==2.3.9 - # via gt4py -h5netcdf==1.6.1 - # via ndsl -h5py==3.13.0 - # via h5netcdf -importlib-metadata==8.7.0 - # via dask -jinja2==3.1.6 - # via gt4py -kiwisolver==1.4.8 - # via matplotlib -lark==1.2.2 - # via gt4py -locket==1.0.0 - # via partd -mako==1.3.10 - # via gt4py -markupsafe==3.0.2 - # via - # jinja2 - # mako -matplotlib==3.10.1 - # via ndsl -mpi4py==3.1.5 - # via ndsl -mpmath==1.3.0 - # via sympy -mypy-extensions==1.1.0 - # via black -nanobind==2.7.0 - # via gt4py -netcdf4==1.7.1 - # via ndsl -networkx==3.4.2 - # via dace -ninja==1.11.1.4 - # via gt4py -numcodecs==0.15.1 - # via zarr -numpy==1.26.4 - # via - # cftime - # contourpy - # dace - # gt4py - # h5py - # matplotlib - # ndsl - # netcdf4 - # numcodecs - # pace (setup.py) - # pandas - # pyfv3 - # pyshield - # scipy - # xarray - # zarr -orderly-set==5.4.0 - # via deepdiff -packaging==25.0 - # via - # black - # dace - # dask - # gt4py - # h5netcdf - # matplotlib - # setuptools-scm - # xarray -pandas==2.2.3 - # via xarray -partd==1.4.2 - # via dask -pathspec==0.12.1 - # via black -pillow==11.2.1 - # via matplotlib -platformdirs==4.3.7 - # via black -ply==3.11 - # via dace -pybind11==2.13.6 - # via gt4py -pygments==2.19.1 - # via devtools -pyparsing==3.2.3 - # via matplotlib -python-dateutil==2.9.0.post0 - # via - # matplotlib - # pandas -pytz==2025.2 - # via pandas -pyyaml==6.0.2 - # via - # dace - # dask -scipy==1.15.2 - # via ndsl -setuptools-scm==8.3.1 - # via fparser -six==1.17.0 - # via - # asttokens - # astunparse - # python-dateutil -sympy==1.14.0 - # via dace -tabulate==0.9.0 - # via gt4py -toolz==1.0.0 - # via - # cytoolz - # dask - # partd -typing-extensions==4.13.2 - # via gt4py -tzdata==2025.2 - # via - # faker - # pandas -wheel==0.45.1 - # via astunparse -wrapt==1.17.2 - # via deprecated -xarray==2025.4.0 - # via - # ndsl - # pace (setup.py) - # pyfv3 - # pyshield -xxhash==3.0.0 - # via gt4py -zarr==2.18.7 - # via pace (setup.py) -zipp==3.21.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/examples/build_scripts/build_gaea_c4.sh b/examples/build_scripts/build_gaea_c4.sh index 55ca940a..608cf9fb 100644 --- a/examples/build_scripts/build_gaea_c4.sh +++ b/examples/build_scripts/build_gaea_c4.sh @@ -23,4 +23,4 @@ conda create -y --name my_name python=3.8 conda activate my_name # install Pace dependencies and its dependencies -pip3 install -c constraints-test.txt .[test] +pip3 install .[test] diff --git a/examples/build_scripts/build_gaea_c5.sh b/examples/build_scripts/build_gaea_c5.sh index 73c4c2b8..627c8a06 100644 --- a/examples/build_scripts/build_gaea_c5.sh +++ b/examples/build_scripts/build_gaea_c5.sh @@ -22,5 +22,5 @@ conda create -y --name my_name python=3.8 # enter the environment conda activate my_name -# install the Pace dependencies, GT4Py, and Pace -pip3 install -c constraints-test.txt .[test] +# install Pace dependencies and its dependencies +pip3 install .[test] diff --git a/examples/build_scripts/build_gaea_c5_gcc.sh b/examples/build_scripts/build_gaea_c5_gcc.sh index aa84c826..782ac5f6 100644 --- a/examples/build_scripts/build_gaea_c5_gcc.sh +++ b/examples/build_scripts/build_gaea_c5_gcc.sh @@ -32,4 +32,4 @@ conda create -y --name my_name python=3.11.7 conda activate my_name # install Pace dependencies and its dependencies -pip3 install -c constraints-test.txt .[test] +pip3 install .[test] diff --git a/examples/build_scripts/build_ppan.sh b/examples/build_scripts/build_ppan.sh index 8c112e2d..ef88d081 100644 --- a/examples/build_scripts/build_ppan.sh +++ b/examples/build_scripts/build_ppan.sh @@ -28,7 +28,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.8 matplotlib==3 conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir -c constraints-test.txt .[test] +pip3 install --no-cache-dir .[test] # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/build_scripts/build_ppan_intel.sh b/examples/build_scripts/build_ppan_intel.sh index d8a81e4b..f3ee3fdd 100644 --- a/examples/build_scripts/build_ppan_intel.sh +++ b/examples/build_scripts/build_ppan_intel.sh @@ -35,7 +35,7 @@ conda create -c conda-forge -y --name $ENVIRONMENT_NAME python=3.11.7 matplotlib conda activate $ENVIRONMENT_NAME # install Pace dependencies and its dependencies -pip3 install --no-cache-dir -c constraints-test.txt .[test] +pip3 install --no-cache-dir .[test] # clone fv3net git clone https://github.com/ai2cm/fv3net.git $FV3NET_DIR diff --git a/examples/create_venv.sh b/examples/create_venv.sh index 68c197e5..f523ae25 100755 --- a/examples/create_venv.sh +++ b/examples/create_venv.sh @@ -12,7 +12,7 @@ python3 -m venv venv rundir=$(pwd) cd ${SCRIPT_DIR}/../ -pip install -c constraints-dev.txt -e . +pip install -e .[dev] deactivate cd $rundir From c761437691b6dce20e0bf8beb1d0e359c108364e Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:10:29 +0200 Subject: [PATCH 16/17] for easier (future) debuggging: print versions of installed packages --- .github/workflows/main_unit_tests.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main_unit_tests.yaml b/.github/workflows/main_unit_tests.yaml index fe214c89..b98df73c 100644 --- a/.github/workflows/main_unit_tests.yaml +++ b/.github/workflows/main_unit_tests.yaml @@ -42,16 +42,17 @@ jobs: submodules: 'recursive' path: pace/${{inputs.component_name}} - - name: Print versions - run: | - python --version - pip --version - - name: Install packages run: | cd ${GITHUB_WORKSPACE}/pace pip install .[test] + - name: Print versions + run: | + python --version + pip --version + pip list + - name: Prepare input files run: | cd ${GITHUB_WORKSPACE}/pace From e50ae13e14afad137c9ba27af8654e3f47d565c0 Mon Sep 17 00:00:00 2001 From: Roman Cattaneo <1116746+romanc@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:13:21 +0200 Subject: [PATCH 17/17] one last (tm) update to the readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e9110db..30e9a0a2 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Inside of your pace `venv` or `conda` environment, install pace and its dependen pip install -e .[dev] ``` -For running tests, we recommend to install pace with pinned dependency versions of the test system to enable reproducibility: +For running tests, we recommend to install pace with the `[test]` extra (avoid pulling other dev dependencies): ```shell pip install .[test]