From 5d35008a66fa3fff4fbffd2c6a57326319bdb73f Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 16 Sep 2021 14:48:17 -0600 Subject: [PATCH 01/10] Add max_instances build attribute to template --- babelizer/data/cookiecutter.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/babelizer/data/cookiecutter.json b/babelizer/data/cookiecutter.json index 3c2432eb..920018d9 100644 --- a/babelizer/data/cookiecutter.json +++ b/babelizer/data/cookiecutter.json @@ -6,7 +6,8 @@ "libraries": [], "library_dirs": [], "include_dirs": [], - "extra_compile_args": [] + "extra_compile_args": [], + "max_instances": 3 }, "info": { "full_name": "CSDMS", From 51df6d509e5c0cc1c0614b507a222c5963b600c0 Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 16 Sep 2021 14:49:12 -0600 Subject: [PATCH 02/10] Norm default max_instances value --- babelizer/metadata.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/babelizer/metadata.py b/babelizer/metadata.py index 03be4d76..20a40ee0 100644 --- a/babelizer/metadata.py +++ b/babelizer/metadata.py @@ -230,6 +230,7 @@ def validate(config): "library_dirs", "include_dirs", "extra_compile_args", + "max_instances", ), ) validate_dict(config["package"], required=("name", "requirements"), optional={}) @@ -313,6 +314,9 @@ def norm(config): except KeyError: pass + if "max_instances" not in build: + build["max_instances"] = 3 + if "entry_point" in config["library"]: libraries = BabelMetadata._handle_old_style_entry_points(config["library"]) else: @@ -335,6 +339,7 @@ def norm(config): "library_dirs": build["library_dirs"], "include_dirs": build["include_dirs"], "extra_compile_args": build["extra_compile_args"], + "max_instances": build["max_instances"], }, "package": { "name": config["package"]["name"], @@ -446,6 +451,7 @@ def as_cookiecutter_context(self): "library_dirs": self._meta["build"]["library_dirs"], "include_dirs": self._meta["build"]["include_dirs"], "extra_compile_args": self._meta["build"]["extra_compile_args"], + "max_instances": self._meta["build"]["max_instances"], }, "info": { "full_name": self._meta["info"]["package_author"], From 5a1ecd84bf6f83d4e37a74688950804235f70514 Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 16 Sep 2021 14:49:48 -0600 Subject: [PATCH 03/10] Use max_instances build attribute in Fortran interoperability code --- .../{{cookiecutter.package_name}}/lib/bmi_interoperability.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/babelizer/data/{{cookiecutter.package_name}}/{{cookiecutter.package_name}}/lib/bmi_interoperability.f90 b/babelizer/data/{{cookiecutter.package_name}}/{{cookiecutter.package_name}}/lib/bmi_interoperability.f90 index 91ba3049..80bf7580 100644 --- a/babelizer/data/{{cookiecutter.package_name}}/{{cookiecutter.package_name}}/lib/bmi_interoperability.f90 +++ b/babelizer/data/{{cookiecutter.package_name}}/{{cookiecutter.package_name}}/lib/bmi_interoperability.f90 @@ -17,7 +17,7 @@ module bmi_interoperability implicit none - integer, parameter :: N_MODELS = 2048 + integer, parameter :: N_MODELS = {{ cookiecutter.build.max_instances }} type ({{ component.entry_point }}) :: model_array(N_MODELS) logical :: model_avail(N_MODELS) = .true. From 617b9df1483d78fe5e2ad97b0e6c79ae2051593a Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 16 Sep 2021 15:38:35 -0600 Subject: [PATCH 04/10] More than 3 models are needed for our tests, apparently --- babelizer/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/babelizer/metadata.py b/babelizer/metadata.py index 20a40ee0..5bee02d0 100644 --- a/babelizer/metadata.py +++ b/babelizer/metadata.py @@ -315,7 +315,7 @@ def norm(config): pass if "max_instances" not in build: - build["max_instances"] = 3 + build["max_instances"] = 8 if "entry_point" in config["library"]: libraries = BabelMetadata._handle_old_style_entry_points(config["library"]) From 8bcec9b25e1c62aa3b58d644751690ea0b0920d4 Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 16 Sep 2021 16:33:26 -0600 Subject: [PATCH 05/10] Update changelog --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index bf4a4452..f5ae220e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,7 @@ Changelog for babelizer ****************** - Complete API documentation (#55) +- Configure max number of model instances (#61) ****************** From 715279543e69fbe2b9c7db3856b1c6b317cb899d Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 16 Sep 2021 16:34:06 -0600 Subject: [PATCH 06/10] Create tests for new max_instances build parameter --- external/tests/test_fortran.py | 35 ++++++++++++++++++- .../tests/test_fortran/user-instances.toml | 30 ++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 external/tests/test_fortran/user-instances.toml diff --git a/external/tests/test_fortran.py b/external/tests/test_fortran.py index 1fc8ba71..04e77a79 100644 --- a/external/tests/test_fortran.py +++ b/external/tests/test_fortran.py @@ -3,13 +3,14 @@ import pathlib import shutil import subprocess +import tomlkit as toml from click.testing import CliRunner from babelizer.cli import babelize -def test_babelize_init_fortran(tmpdir, datadir): +def test_babelize_init_fortran_default_instances(tmpdir, datadir): runner = CliRunner() with tmpdir.as_cwd(): @@ -20,6 +21,38 @@ def test_babelize_init_fortran(tmpdir, datadir): assert pathlib.Path("pymt_heatf").exists() assert (pathlib.Path("pymt_heatf") / "babel.toml").is_file() + with open("pymt_heatf/babel.toml", "r") as fp: + meta = toml.parse(fp.read()) + + assert "max_instances" in meta["build"] + assert meta["build"]["max_instances"] == 8 + + +def test_babelize_init_fortran_user_instances(tmpdir, datadir): + runner = CliRunner() + + with tmpdir.as_cwd(): + shutil.copy(datadir / "user-instances.toml", ".") + result = runner.invoke(babelize, ["init", "user-instances.toml"]) + + assert result.exit_code == 0 + assert pathlib.Path("pymt_heatf").exists() + assert (pathlib.Path("pymt_heatf") / "babel.toml").is_file() + + with open("pymt_heatf/babel.toml", "r") as fp: + meta = toml.parse(fp.read()) + + assert "max_instances" in meta["build"] + assert meta["build"]["max_instances"] == 32 + + +def test_babelize_build_fortran_example(tmpdir, datadir): + runner = CliRunner() + + with tmpdir.as_cwd(): + shutil.copy(datadir / "babel.toml", ".") + result = runner.invoke(babelize, ["init", "babel.toml"]) + try: result = subprocess.run( ["pip", "install", "-e", "."], diff --git a/external/tests/test_fortran/user-instances.toml b/external/tests/test_fortran/user-instances.toml new file mode 100644 index 00000000..cc061912 --- /dev/null +++ b/external/tests/test_fortran/user-instances.toml @@ -0,0 +1,30 @@ +[library] +[library.HeatBMI] +language = "fortran" +library = "bmiheatf" +header = "" +entry_point = "bmi_heat" + +[build] +undef_macros = [] +define_macros = [] +libraries = [] +library_dirs = [] +include_dirs = [] +extra_compile_args = [] +max_instances = 32 + +[package] +name = "pymt_heatf" +requirements = [""] + +[info] +github_username = "pymt-lab" +package_author = "csdms" +package_author_email = "csdms@colorado.edu" +package_license = "MIT" +summary = "PyMT plugin for heatf model" + +[ci] +python_version = ["3.9"] +os = ["linux", "mac", "windows"] From 292db13cb96fb598096c12293b80dbe1cbb9e5bb Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 21 Oct 2021 14:39:58 -0600 Subject: [PATCH 07/10] Remove unused babelizer config file --- .../tests/test_fortran/user-instances.toml | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 external/tests/test_fortran/user-instances.toml diff --git a/external/tests/test_fortran/user-instances.toml b/external/tests/test_fortran/user-instances.toml deleted file mode 100644 index cc061912..00000000 --- a/external/tests/test_fortran/user-instances.toml +++ /dev/null @@ -1,30 +0,0 @@ -[library] -[library.HeatBMI] -language = "fortran" -library = "bmiheatf" -header = "" -entry_point = "bmi_heat" - -[build] -undef_macros = [] -define_macros = [] -libraries = [] -library_dirs = [] -include_dirs = [] -extra_compile_args = [] -max_instances = 32 - -[package] -name = "pymt_heatf" -requirements = [""] - -[info] -github_username = "pymt-lab" -package_author = "csdms" -package_author_email = "csdms@colorado.edu" -package_license = "MIT" -summary = "PyMT plugin for heatf model" - -[ci] -python_version = ["3.9"] -os = ["linux", "mac", "windows"] From 1fae6cbb2bb0c0706f04f26a4089b1874efa2058 Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 21 Oct 2021 14:43:33 -0600 Subject: [PATCH 08/10] Parameterize max_instances in test and mark init test as dependency --- external/tests/test_fortran.py | 61 ++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/external/tests/test_fortran.py b/external/tests/test_fortran.py index 04e77a79..9510449b 100644 --- a/external/tests/test_fortran.py +++ b/external/tests/test_fortran.py @@ -4,37 +4,48 @@ import shutil import subprocess import tomlkit as toml +import pytest from click.testing import CliRunner from babelizer.cli import babelize -def test_babelize_init_fortran_default_instances(tmpdir, datadir): - runner = CliRunner() - +@pytest.mark.dependency() +def test_babelize_init_works(tmpdir, datadir): + runner = CliRunner(mix_stderr=False) with tmpdir.as_cwd(): - shutil.copy(datadir / "babel.toml", ".") - result = runner.invoke(babelize, ["init", "babel.toml"]) + result = runner.invoke(babelize, ["init", str(datadir / "babel.toml")]) + assert result.stdout.strip() == str(tmpdir / "pymt_heatf") assert result.exit_code == 0 assert pathlib.Path("pymt_heatf").exists() assert (pathlib.Path("pymt_heatf") / "babel.toml").is_file() - with open("pymt_heatf/babel.toml", "r") as fp: - meta = toml.parse(fp.read()) + repo = git.Repo("pymt_heatf") - assert "max_instances" in meta["build"] - assert meta["build"]["max_instances"] == 8 + assert not repo.bare -def test_babelize_init_fortran_user_instances(tmpdir, datadir): - runner = CliRunner() +@pytest.mark.dependency(depends=["test_babelize_init_works"]) +@pytest.mark.parametrize("max_instances", [8, 32, None]) +def test_babelize_init_fortran_with_user_instances(tmpdir, datadir, max_instances): + runner = CliRunner(mix_stderr=False) + + babel_toml = toml.load(datadir / "babel.toml") + babel_toml["build"]["max_instances"] = max_instances + + if max_instances is None: + babel_toml["build"].pop("max_instances", None) + max_instances = 8 with tmpdir.as_cwd(): - shutil.copy(datadir / "user-instances.toml", ".") - result = runner.invoke(babelize, ["init", "user-instances.toml"]) + with open("babel.toml", "w") as fp: + toml.dump(babel_toml, fp) + result = runner.invoke(babelize, ["init", "babel.toml"]) + + assert result.stdout.strip() == str(tmpdir / "pymt_heatf") assert result.exit_code == 0 assert pathlib.Path("pymt_heatf").exists() assert (pathlib.Path("pymt_heatf") / "babel.toml").is_file() @@ -43,16 +54,17 @@ def test_babelize_init_fortran_user_instances(tmpdir, datadir): meta = toml.parse(fp.read()) assert "max_instances" in meta["build"] - assert meta["build"]["max_instances"] == 32 + assert meta["build"]["max_instances"] == max_instances +@pytest.mark.dependency(depends=["test_babelize_init_works"]) def test_babelize_build_fortran_example(tmpdir, datadir): - runner = CliRunner() + runner = CliRunner(mix_stderr=False) with tmpdir.as_cwd(): - shutil.copy(datadir / "babel.toml", ".") - result = runner.invoke(babelize, ["init", "babel.toml"]) + runner.invoke(babelize, ["init", str(datadir / "babel.toml")]) + with tmpdir.as_cwd(): try: result = subprocess.run( ["pip", "install", "-e", "."], @@ -89,3 +101,18 @@ def test_babelize_build_fortran_example(tmpdir, datadir): assert err.output is None, err.output assert result.returncode == 0 + + +@pytest.mark.dependency(depends=["test_babelize_init_works"]) +def test_babelize_update_fortran(tmpdir, datadir): + runner = CliRunner(mix_stderr=False) + + with tmpdir.as_cwd(): + runner.invoke(babelize, ["init", str(datadir / "babel.toml")]) + + with tmpdir.as_cwd(): + result = runner.invoke(babelize, ["--cd", "pymt_heatf", "update"]) + + assert result.exit_code == 0 + assert pathlib.Path("pymt_heatf").exists() + assert "re-rendering" in result.stderr From 8f925c1bbdfb19b4274536b356ef5f8fe2129257 Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 21 Oct 2021 14:59:40 -0600 Subject: [PATCH 09/10] Add missing git import --- external/tests/test_fortran.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/external/tests/test_fortran.py b/external/tests/test_fortran.py index 9510449b..9814aa7c 100644 --- a/external/tests/test_fortran.py +++ b/external/tests/test_fortran.py @@ -3,9 +3,10 @@ import pathlib import shutil import subprocess -import tomlkit as toml -import pytest +import git +import pytest +import tomlkit as toml from click.testing import CliRunner from babelizer.cli import babelize From 1ff7b60cb9c2e1ba0653f2956a8a436fe5bf5aa4 Mon Sep 17 00:00:00 2001 From: Mark Piper Date: Thu, 21 Oct 2021 15:12:14 -0600 Subject: [PATCH 10/10] Use toml package instead of tomlkit --- external/tests/test_fortran.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/external/tests/test_fortran.py b/external/tests/test_fortran.py index 9814aa7c..b5a7a518 100644 --- a/external/tests/test_fortran.py +++ b/external/tests/test_fortran.py @@ -6,7 +6,7 @@ import git import pytest -import tomlkit as toml +import toml from click.testing import CliRunner from babelizer.cli import babelize @@ -51,8 +51,7 @@ def test_babelize_init_fortran_with_user_instances(tmpdir, datadir, max_instance assert pathlib.Path("pymt_heatf").exists() assert (pathlib.Path("pymt_heatf") / "babel.toml").is_file() - with open("pymt_heatf/babel.toml", "r") as fp: - meta = toml.parse(fp.read()) + meta = toml.load("pymt_heatf/babel.toml") assert "max_instances" in meta["build"] assert meta["build"]["max_instances"] == max_instances